You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2018/04/26 20:32:52 UTC
[01/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Repository: storm
Updated Branches:
refs/heads/master 767325340 -> 4137328b7
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestResourceAwareScheduler.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestResourceAwareScheduler.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestResourceAwareScheduler.java
index 85fb569..45a2436 100644
--- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestResourceAwareScheduler.java
+++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestResourceAwareScheduler.java
@@ -1,24 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource;
-import org.apache.storm.scheduler.resource.normalization.NormalizedResources;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -42,6 +35,7 @@ import org.apache.storm.scheduler.SupervisorDetails;
import org.apache.storm.scheduler.Topologies;
import org.apache.storm.scheduler.TopologyDetails;
import org.apache.storm.scheduler.WorkerSlot;
+import org.apache.storm.scheduler.resource.normalization.NormalizedResources;
import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy;
import org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy;
import org.apache.storm.testing.TestWordCounter;
@@ -59,16 +53,31 @@ import org.junit.rules.ExpectedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.*;
-import static org.junit.Assert.*;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestBolt;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.TestSpout;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.addTopologies;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.assertTopologiesNotScheduled;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createClusterConfig;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genExecsAndComps;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.getSupervisorToCpuUsage;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.getSupervisorToMemoryUsage;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userRes;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userResourcePool;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
public class TestResourceAwareScheduler {
private static final Logger LOG = LoggerFactory.getLogger(TestResourceAwareScheduler.class);
-
- private static int currentTime = 1450418597;
-
private static final Config defaultTopologyConf = createClusterConfig(10, 128, 0, null);
+ private static int currentTime = 1450418597;
+ @Rule
+ public final ExpectedException schedulerException = ExpectedException.none();
@BeforeClass
public static void initConf() {
@@ -287,7 +296,8 @@ public class TestResourceAwareScheduler {
TopologyBuilder builder1 = new TopologyBuilder(); // a topology with multiple spouts
builder1.setSpout("wordSpout", new TestWordSpout(), 2).setCPULoad(250.0).setMemoryLoad(1000.0, 200.0);
- builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(100.0).setMemoryLoad(500.0, 100.0);
+ builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(100.0)
+ .setMemoryLoad(500.0, 100.0);
StormTopology stormTopology1 = builder1.createTopology();
Config config = new Config();
@@ -339,7 +349,7 @@ public class TestResourceAwareScheduler {
Double supervisorTotalMemory = entry.getKey().getTotalMemory();
Double supervisorUsedCpu = 0.0;
Double supervisorUsedMemory = 0.0;
- for (ExecutorDetails executor: entry.getValue()) {
+ for (ExecutorDetails executor : entry.getValue()) {
supervisorUsedMemory += topology1.getTotalCpuReqTask(executor);
supervisorTotalCpu += topology1.getTotalMemReqTask(executor);
}
@@ -359,10 +369,10 @@ public class TestResourceAwareScheduler {
assertEquals(1200.0, assignedExecutorMemory.get(2), 0.001);
for (Map.Entry<Double, Double> entry : memoryAvailableToUsed.entrySet()) {
- assertTrue(entry.getKey()- entry.getValue() >= 0);
+ assertTrue(entry.getKey() - entry.getValue() >= 0);
}
for (Map.Entry<Double, Double> entry : cpuAvailableToUsed.entrySet()) {
- assertTrue(entry.getKey()- entry.getValue() >= 0);
+ assertTrue(entry.getKey() - entry.getValue() >= 0);
}
assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology1.getId()));
}
@@ -592,13 +602,14 @@ public class TestResourceAwareScheduler {
Double cpuUsed = superToCpu.get(supervisor);
Double memUsed = superToMem.get(supervisor);
- assertTrue(supervisor.getId() + " MEM: "+ memAvailable + " == " + memUsed + " OR CPU: " + cpuAvailable + " == " + cpuUsed,
- (Math.abs(memAvailable - memUsed) < EPSILON) || (Math.abs(cpuAvailable - cpuUsed) < EPSILON));
+ assertTrue(supervisor.getId() + " MEM: " + memAvailable + " == " + memUsed + " OR CPU: " + cpuAvailable + " == " + cpuUsed,
+ (Math.abs(memAvailable - memUsed) < EPSILON) || (Math.abs(cpuAvailable - cpuUsed) < EPSILON));
}
// end of Test1
LOG.warn("\n\n\t\tSwitching to topologies 1, 2 and 4");
- // Test2: Launch topo 1, 2 and 4, they together request a little more mem than available, so one of the 3 topos will not be scheduled
+ // Test2: Launch topo 1, 2 and 4, they together request a little more mem than available, so one of the 3 topos will not be
+ // scheduled
topologies = new Topologies(topology1, topology2, topology4);
cluster = new Cluster(iNimbus, supMap, new HashMap<>(), topologies, config1);
rs.prepare(config1);
@@ -675,7 +686,8 @@ public class TestResourceAwareScheduler {
// Test2: test when no more workers are available due to topology worker max heap size limit but there is memory is still available
// wordSpout2 is going to contain 5 executors that needs scheduling. Each of those executors has a memory requirement of 128.0 MB
// The cluster contains 4 free WorkerSlots. For this topolology each worker is limited to a max heap size of 128.0
- // Thus, one executor not going to be able to get scheduled thus failing the scheduling of this topology and no executors of this topology will be scheduleded
+ // Thus, one executor not going to be able to get scheduled thus failing the scheduling of this topology and no executors of this
+ // topology will be scheduleded
TopologyBuilder builder2 = new TopologyBuilder();
builder2.setSpout("wordSpout2", new TestWordSpout(), 5);
StormTopology stormTopology2 = builder2.createTopology();
@@ -717,7 +729,7 @@ public class TestResourceAwareScheduler {
genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"),
genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 20, "bobby"));
Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<>(), topologies, config);
-
+
ResourceAwareScheduler rs = new ResourceAwareScheduler();
rs.prepare(config);
rs.schedule(topologies, cluster);
@@ -753,13 +765,13 @@ public class TestResourceAwareScheduler {
genTopology("topo-14", config, 5, 15, 1, 1, currentTime - 16, 20, "derek"),
genTopology("topo-15", config, 5, 15, 1, 1, currentTime - 24, 29, "derek"));
Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<>(), topologies, config);
-
+
ResourceAwareScheduler rs = new ResourceAwareScheduler();
rs.prepare(config);
rs.schedule(topologies, cluster);
- for (TopologyDetails td: topologies) {
- assertTopologiesFullyScheduled(cluster, td.getName());
+ for (TopologyDetails td : topologies) {
+ assertTopologiesFullyScheduled(cluster, td.getName());
}
}
@@ -812,7 +824,7 @@ public class TestResourceAwareScheduler {
rs.prepare(config);
rs.schedule(topologies, cluster);
-
+
assertTopologiesFullyScheduled(cluster, "topo-1", "topo-2", "topo-3", "topo-4", "topo-5", "topo-6");
//fail supervisor
@@ -838,11 +850,11 @@ public class TestResourceAwareScheduler {
cluster = new Cluster(iNimbus, supMap, newAssignments, topologies, config);
cluster.setStatusMap(statusMap);
rs.schedule(topologies, cluster);
-
+
assertTopologiesFullyScheduled(cluster, "topo-2", "topo-3", "topo-4", "topo-5", "topo-6");
assertTopologiesNotScheduled(cluster, "topo-1");
}
-
+
/**
* test if free slots on nodes work correctly
*/
@@ -876,8 +888,8 @@ public class TestResourceAwareScheduler {
nodes.get(ws.getNodeId()).free(ws);
double memoryAfter = nodes.get(ws.getNodeId()).getAvailableMemoryResources();
double cpuAfter = nodes.get(ws.getNodeId()).getAvailableCpuResources();
- assertEquals("Check if free correctly frees amount of memory", memoryBefore + memoryUsedByWorker, memoryAfter, 0.001);
- assertEquals("Check if free correctly frees amount of memory", cpuBefore + cpuUsedByWorker, cpuAfter, 0.001);
+ assertEquals("Check if free correctly frees amount of memory", memoryBefore + memoryUsedByWorker, memoryAfter, 0.001);
+ assertEquals("Check if free correctly frees amount of memory", cpuBefore + cpuUsedByWorker, cpuAfter, 0.001);
assertFalse("Check if worker was removed from assignments", assignment.getSlotToExecutors().containsKey(ws));
}
}
@@ -891,7 +903,7 @@ public class TestResourceAwareScheduler {
INimbus iNimbus = new INimbusTest();
Map<String, SupervisorDetails> supMap = genSupervisors(8, 4, 100, 1000);
Config config = createClusterConfig(100, 500, 500, null);
-
+
TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, currentTime - 2, 10, "jerry");
TopologyDetails topo2 = genTopology("topo-2", config, 2, 0, 2, 0, currentTime - 2, 20, "jerry");
TopologyDetails topo3 = genTopology("topo-3", config, 1, 2, 1, 1, currentTime - 2, 20, "jerry");
@@ -918,15 +930,15 @@ public class TestResourceAwareScheduler {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("spout-1", new TestSpout(),
- 5);
+ 5);
builder.setSpout("spout-2", new TestSpout(),
- 5);
+ 5);
builder.setBolt("bolt-1", new TestBolt(),
- 5).shuffleGrouping("spout-1").shuffleGrouping("bolt-3");
+ 5).shuffleGrouping("spout-1").shuffleGrouping("bolt-3");
builder.setBolt("bolt-2", new TestBolt(),
- 5).shuffleGrouping("bolt-1");
+ 5).shuffleGrouping("bolt-1");
builder.setBolt("bolt-3", new TestBolt(),
- 5).shuffleGrouping("bolt-2").shuffleGrouping("spout-2");
+ 5).shuffleGrouping("bolt-2").shuffleGrouping("spout-2");
INimbus iNimbus = new INimbusTest();
Map<String, SupervisorDetails> supMap = genSupervisors(25, 1, 100, 1000);
@@ -935,7 +947,7 @@ public class TestResourceAwareScheduler {
StormTopology stormTopology = builder.createTopology();
config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
TopologyDetails topo = new TopologyDetails("topo-1", config, stormTopology,
- 0, genExecsAndComps(stormTopology), 0, "jerry");
+ 0, genExecsAndComps(stormTopology), 0, "jerry");
Topologies topologies = new Topologies(topo);
Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
@@ -948,9 +960,6 @@ public class TestResourceAwareScheduler {
assertEquals("Topo all executors scheduled?", 25, cluster.getAssignmentById(topo.getId()).getExecutorToSlot().size());
}
- @Rule
- public final ExpectedException schedulerException = ExpectedException.none();
-
@Test
public void testSchedulerStrategyWhitelist() {
Map<String, Object> config = ConfigUtils.readStormConfig();
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUser.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUser.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUser.java
index e4ed4dc..2876c7d 100644
--- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUser.java
+++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUser.java
@@ -1,24 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource;
-import java.util.Iterator;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.scheduler.Cluster;
import org.apache.storm.scheduler.INimbus;
@@ -28,18 +23,17 @@ import org.apache.storm.scheduler.TopologyDetails;
import org.apache.storm.scheduler.WorkerSlot;
import org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.INimbusTest;
import org.apache.storm.utils.Time;
-import org.apache.storm.utils.Utils;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.*;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.createClusterConfig;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genSupervisors;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.genTopology;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.toDouble;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userRes;
+import static org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler.userResourcePool;
public class TestUser {
private static final Logger LOG = LoggerFactory.getLogger(TestUser.class);
@@ -64,7 +58,9 @@ public class TestUser {
Assert.assertEquals("check cpu resource guarantee", cpuGuarantee, user1.getCpuResourceGuaranteed(), 0.001);
Assert.assertEquals("check memory resource guarantee", memoryGuarantee, user1.getMemoryResourceGuaranteed(), 0.001);
- Assert.assertEquals("check cpu resource pool utilization", ((100.0 * 3.0) / cpuGuarantee), user1.getCpuResourcePoolUtilization(cluster), 0.001);
- Assert.assertEquals("check memory resource pool utilization", ((200.0 + 200.0) * 3.0) / memoryGuarantee, user1.getMemoryResourcePoolUtilization(cluster), 0.001);
+ Assert.assertEquals("check cpu resource pool utilization", ((100.0 * 3.0) / cpuGuarantee),
+ user1.getCpuResourcePoolUtilization(cluster), 0.001);
+ Assert.assertEquals("check memory resource pool utilization", ((200.0 + 200.0) * 3.0) / memoryGuarantee,
+ user1.getMemoryResourcePoolUtilization(cluster), 0.001);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUtilsForResourceAwareScheduler.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUtilsForResourceAwareScheduler.java b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUtilsForResourceAwareScheduler.java
index f534f2e..222c0cd 100644
--- a/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUtilsForResourceAwareScheduler.java
+++ b/storm-server/src/test/java/org/apache/storm/scheduler/resource/TestUtilsForResourceAwareScheduler.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource;
@@ -92,7 +86,7 @@ public class TestUtilsForResourceAwareScheduler {
public static Map<String, Double> toDouble(Map<String, Number> resources) {
Map<String, Double> ret = new HashMap<>();
- for (Entry<String, Number> entry: resources.entrySet()) {
+ for (Entry<String, Number> entry : resources.entrySet()) {
ret.put(entry.getKey(), entry.getValue().doubleValue());
}
return ret;
@@ -100,21 +94,22 @@ public class TestUtilsForResourceAwareScheduler {
public static Map<String, Map<String, Number>> userResourcePool(TestUserResources... resources) {
Map<String, Map<String, Number>> ret = new HashMap<>();
- for (TestUserResources res: resources) {
+ for (TestUserResources res : resources) {
res.addSelfTo(ret);
}
return ret;
}
public static Config createGrasClusterConfig(double compPcore, double compOnHeap, double compOffHeap,
- Map<String, Map<String, Number>> pools, Map<String, Double> genericResourceMap) {
+ Map<String, Map<String, Number>> pools, Map<String, Double> genericResourceMap) {
Config config = createClusterConfig(compPcore, compOnHeap, compOffHeap, pools);
config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, genericResourceMap);
config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, GenericResourceAwareStrategy.class.getName());
return config;
}
- public static Config createClusterConfig(double compPcore, double compOnHeap, double compOffHeap, Map<String, Map<String, Number>> pools) {
+ public static Config createClusterConfig(double compPcore, double compOnHeap, double compOffHeap,
+ Map<String, Map<String, Number>> pools) {
Config config = new Config();
config.putAll(Utils.readDefaultConfig());
config.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, DefaultSchedulingPriorityStrategy.class.getName());
@@ -144,17 +139,18 @@ public class TestUtilsForResourceAwareScheduler {
public static Map<String, SupervisorDetails> genSupervisors(int numSup, int numPorts, int start,
double cpu, double mem, Map<String, Double> miscResources) {
- Map<String, Double> resourceMap = new HashMap<>();
- resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, cpu);
- resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, mem);
- resourceMap.putAll(miscResources);
+ Map<String, Double> resourceMap = new HashMap<>();
+ resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, cpu);
+ resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, mem);
+ resourceMap.putAll(miscResources);
Map<String, SupervisorDetails> retList = new HashMap<String, SupervisorDetails>();
for (int i = start; i < numSup + start; i++) {
List<Number> ports = new LinkedList<>();
for (int j = 0; j < numPorts; j++) {
ports.add(j);
}
- SupervisorDetails sup = new SupervisorDetails("sup-" + i, "host-" + i, null, ports, NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceMap));
+ SupervisorDetails sup = new SupervisorDetails("sup-" + i, "host-" + i, null, ports,
+ NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceMap));
retList.put(sup.getId(), sup);
}
return retList;
@@ -188,12 +184,12 @@ public class TestUtilsForResourceAwareScheduler {
return retMap;
}
- public static Topologies addTopologies(Topologies topos, TopologyDetails ... details) {
+ public static Topologies addTopologies(Topologies topos, TopologyDetails... details) {
Map<String, TopologyDetails> topoMap = new HashMap<>();
- for (TopologyDetails td: topos.getTopologies()) {
+ for (TopologyDetails td : topos.getTopologies()) {
topoMap.put(td.getId(), td);
}
- for (TopologyDetails td: details) {
+ for (TopologyDetails td : details) {
if (topoMap.put(td.getId(), td) != null) {
throw new IllegalArgumentException("Cannot have multiple topologies with id " + td.getId());
}
@@ -213,21 +209,21 @@ public class TestUtilsForResourceAwareScheduler {
conf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
StormTopology topology = buildTopology(numSpout, numBolt, spoutParallelism, boltParallelism);
TopologyDetails topo = new TopologyDetails(name + "-" + launchTime, conf, topology,
- 0,
- genExecsAndComps(topology), launchTime, user);
+ 0,
+ genExecsAndComps(topology), launchTime, user);
return topo;
}
public static StormTopology buildTopology(int numSpout, int numBolt,
int spoutParallelism, int boltParallelism) {
LOG.debug("buildTopology with -> numSpout: " + numSpout + " spoutParallelism: "
- + spoutParallelism + " numBolt: "
- + numBolt + " boltParallelism: " + boltParallelism);
+ + spoutParallelism + " numBolt: "
+ + numBolt + " boltParallelism: " + boltParallelism);
TopologyBuilder builder = new TopologyBuilder();
for (int i = 0; i < numSpout; i++) {
SpoutDeclarer s1 = builder.setSpout("spout-" + i, new TestSpout(),
- spoutParallelism);
+ spoutParallelism);
}
int j = 0;
for (int i = 0; i < numBolt; i++) {
@@ -235,7 +231,7 @@ public class TestUtilsForResourceAwareScheduler {
j = 0;
}
BoltDeclarer b1 = builder.setBolt("bolt-" + i, new TestBolt(),
- boltParallelism).shuffleGrouping("spout-" + j);
+ boltParallelism).shuffleGrouping("spout-" + j);
j++;
}
@@ -263,7 +259,7 @@ public class TestUtilsForResourceAwareScheduler {
public void nextTuple() {
Utils.sleep(100);
- final String[] words = new String[]{"nathan", "mike", "jackson", "golda", "bertels"};
+ final String[] words = new String[]{ "nathan", "mike", "jackson", "golda", "bertels" };
final Random rand = new Random();
final String word = words[rand.nextInt(words.length)];
_collector.emit(new Values(word));
@@ -349,34 +345,34 @@ public class TestUtilsForResourceAwareScheduler {
return m.find();
}
- public static void assertTopologiesNotScheduled(Cluster cluster, String ... topoNames) {
+ public static void assertTopologiesNotScheduled(Cluster cluster, String... topoNames) {
Topologies topologies = cluster.getTopologies();
- for (String topoName: topoNames) {
+ for (String topoName : topoNames) {
TopologyDetails td = topologies.getByName(topoName);
- assert(td != null) : topoName;
+ assert (td != null) : topoName;
String topoId = td.getId();
String status = cluster.getStatus(topoId);
- assert(status != null) : topoName;
- assert(!isStatusSuccess(status)) : topoName;
- assert(cluster.getAssignmentById(topoId) == null) : topoName;
- assert(cluster.needsSchedulingRas(td)) : topoName;
+ assert (status != null) : topoName;
+ assert (!isStatusSuccess(status)) : topoName;
+ assert (cluster.getAssignmentById(topoId) == null) : topoName;
+ assert (cluster.needsSchedulingRas(td)) : topoName;
}
}
- public static void assertTopologiesFullyScheduled(Cluster cluster, String ... topoNames) {
+ public static void assertTopologiesFullyScheduled(Cluster cluster, String... topoNames) {
Topologies topologies = cluster.getTopologies();
- for (String topoName: topoNames) {
+ for (String topoName : topoNames) {
TopologyDetails td = topologies.getByName(topoName);
- assert(td != null) : topoName;
+ assert (td != null) : topoName;
String topoId = td.getId();
assertStatusSuccess(cluster, topoId);
- assert(cluster.getAssignmentById(topoId) != null) : topoName;
- assert(cluster.needsSchedulingRas(td) == false) : topoName;
+ assert (cluster.getAssignmentById(topoId) != null) : topoName;
+ assert (cluster.needsSchedulingRas(td) == false) : topoName;
}
}
public static void assertStatusSuccess(Cluster cluster, String topoId) {
- assert(isStatusSuccess(cluster.getStatus(topoId))) :
+ assert (isStatusSuccess(cluster.getStatus(topoId))) :
"topology status " + topoId + " is not successful " + cluster.getStatus(topoId);
}
@@ -409,7 +405,7 @@ public class TestUtilsForResourceAwareScheduler {
}
for (Map.Entry<SupervisorDetails, List<ExecutorDetails>> entry : supervisorToExecutors.entrySet()) {
Double supervisorUsedMemory = 0.0;
- for (ExecutorDetails executor: entry.getValue()) {
+ for (ExecutorDetails executor : entry.getValue()) {
supervisorUsedMemory += topology.getTotalMemReqTask(executor);
}
superToMem.put(entry.getKey(), superToMem.get(entry.getKey()) + supervisorUsedMemory);
@@ -443,7 +439,7 @@ public class TestUtilsForResourceAwareScheduler {
}
for (Map.Entry<SupervisorDetails, List<ExecutorDetails>> entry : supervisorToExecutors.entrySet()) {
Double supervisorUsedCpu = 0.0;
- for (ExecutorDetails executor: entry.getValue()) {
+ for (ExecutorDetails executor : entry.getValue()) {
supervisorUsedCpu += topology.getTotalCpuReqTask(executor);
}
superToCpu.put(entry.getKey(), superToCpu.get(entry.getKey()) + supervisorUsedCpu);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderTest.java
index 60f8701..6003dcf 100644
--- a/storm-server/src/test/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderTest.java
+++ b/storm-server/src/test/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderTest.java
@@ -1,70 +1,39 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.utils;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Time.SimulatedTime;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.Test;
import org.junit.Before;
-import org.junit.After;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.nio.file.Path;
-import java.nio.file.Files;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.storm.utils.Time.SimulatedTime;
public class ArtifactoryConfigLoaderTest {
private static final Logger LOG = LoggerFactory.getLogger(ArtifactoryConfigLoaderTest.class);
- private Path tmpDirPath;
private static final String ARTIFACTORY_HTTP_SCHEME_PREFIX = "artifactory+http://";
-
- private class ArtifactoryConfigLoaderMock extends ArtifactoryConfigLoader {
- String getData;
- HashMap<String, String> getDataMap = new HashMap<>();
-
- public ArtifactoryConfigLoaderMock(Map<String, Object> conf) {
- super(conf);
- }
-
- public void setData(String api, String artifact, String data) {
- if (api == null) {
- getData = data;
- } else {
- getDataMap.put(artifact, data);
- }
- }
-
- @Override
- protected String doGet(String api, String artifact, String host, Integer port) {
- if (api == null) {
- return getData;
- }
- return getDataMap.get(artifact);
- }
- };
+ private Path tmpDirPath;
@Before
public void createTempDir() throws Exception {
@@ -75,6 +44,8 @@ public class ArtifactoryConfigLoaderTest {
dir.mkdir();
}
+ ;
+
@After
public void removeTempDir() throws Exception {
FileUtils.deleteDirectory(tmpDirPath.toFile());
@@ -92,14 +63,16 @@ public class ArtifactoryConfigLoaderTest {
public void testPointingAtDirectory() {
// This is a test where we are configured to point right at an artifact dir
Config conf = new Config();
- conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI, ARTIFACTORY_HTTP_SCHEME_PREFIX + "bogushost.yahoo.com:9999/location/of/this/dir");
+ conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI,
+ ARTIFACTORY_HTTP_SCHEME_PREFIX + "bogushost.yahoo.com:9999/location/of/this/dir");
conf.put(Config.STORM_LOCAL_DIR, tmpDirPath.toString());
ArtifactoryConfigLoaderMock loaderMock = new ArtifactoryConfigLoaderMock(conf);
loaderMock.setData("Anything", "/location/of/this/dir",
- "{\"children\" : [ { \"uri\" : \"/20160621204337.yaml\", \"folder\" : false }]}" );
- loaderMock.setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + "\": {one: 1, two: 2, three: 3, four : 4}}");
+ "{\"children\" : [ { \"uri\" : \"/20160621204337.yaml\", \"folder\" : false }]}");
+ loaderMock
+ .setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + "\": {one: 1, two: 2, three: 3, four : 4}}");
Map<String, Object> ret = loaderMock.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS);
Assert.assertNotNull("Unexpectedly returned null", ret);
@@ -123,14 +96,15 @@ public class ArtifactoryConfigLoaderTest {
public void testArtifactUpdate() {
// This is a test where we are configured to point right at an artifact dir
Config conf = new Config();
- conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI, ARTIFACTORY_HTTP_SCHEME_PREFIX + "bogushost.yahoo.com:9999/location/of/test/dir");
+ conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI,
+ ARTIFACTORY_HTTP_SCHEME_PREFIX + "bogushost.yahoo.com:9999/location/of/test/dir");
conf.put(Config.STORM_LOCAL_DIR, tmpDirPath.toString());
try (SimulatedTime t = new SimulatedTime()) {
ArtifactoryConfigLoaderMock loaderMock = new ArtifactoryConfigLoaderMock(conf);
loaderMock.setData("Anything", "/location/of/test/dir",
- "{\"children\" : [ { \"uri\" : \"/20160621204337.yaml\", \"folder\" : false }]}" );
+ "{\"children\" : [ { \"uri\" : \"/20160621204337.yaml\", \"folder\" : false }]}");
loaderMock.setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + "\": {one: 1, two: 2, three: 3}}");
Map<String, Object> ret = loaderMock.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS);
@@ -151,8 +125,9 @@ public class ArtifactoryConfigLoaderTest {
// Now let's update it, but not advance time. Should get old map again.
loaderMock.setData("Anything", "/location/of/test/dir",
- "{\"children\" : [ { \"uri\" : \"/20160621204999.yaml\", \"folder\" : false }]}");
- loaderMock.setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + "\": {one: 1, two: 2, three: 3, four : 4}}");
+ "{\"children\" : [ { \"uri\" : \"/20160621204999.yaml\", \"folder\" : false }]}");
+ loaderMock
+ .setData(null, null, "{ \"" + DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS + "\": {one: 1, two: 2, three: 3, four : 4}}");
ret = loaderMock.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS);
Assert.assertNotNull("Unexpectedly returned null", ret);
Assert.assertEquals(1, ret.get("one"));
@@ -169,7 +144,7 @@ public class ArtifactoryConfigLoaderTest {
Assert.assertNull("Unexpectedly did not return null, last cached result should not have \"four\"", ret2.get("four"));
// Now, let's advance time.
- Time.advanceTime(11*60*1000);
+ Time.advanceTime(11 * 60 * 1000);
ret = loaderMock.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS);
Assert.assertNotNull("Unexpectedly returned null", ret);
@@ -192,7 +167,8 @@ public class ArtifactoryConfigLoaderTest {
public void testPointingAtSpecificArtifact() {
// This is a test where we are configured to point right at a single artifact
Config conf = new Config();
- conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI, ARTIFACTORY_HTTP_SCHEME_PREFIX + "bogushost.yahoo.com:9999/location/of/this/artifact");
+ conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI,
+ ARTIFACTORY_HTTP_SCHEME_PREFIX + "bogushost.yahoo.com:9999/location/of/this/artifact");
conf.put(Config.STORM_LOCAL_DIR, tmpDirPath.toString());
ArtifactoryConfigLoaderMock loaderMock = new ArtifactoryConfigLoaderMock(conf);
@@ -219,7 +195,8 @@ public class ArtifactoryConfigLoaderTest {
public void testMalformedYaml() throws Exception {
// This is a test where we are configured to point right at a single artifact
Config conf = new Config();
- conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI, ARTIFACTORY_HTTP_SCHEME_PREFIX + "bogushost.yahoo.com:9999/location/of/this/artifact");
+ conf.put(DaemonConfig.SCHEDULER_CONFIG_LOADER_URI,
+ ARTIFACTORY_HTTP_SCHEME_PREFIX + "bogushost.yahoo.com:9999/location/of/this/artifact");
conf.put(Config.STORM_LOCAL_DIR, tmpDirPath.toString());
ArtifactoryConfigLoaderMock loaderMock = new ArtifactoryConfigLoaderMock(conf);
@@ -229,4 +206,29 @@ public class ArtifactoryConfigLoaderTest {
Map<String, Object> ret = loaderMock.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS);
Assert.assertNull("Unexpectedly returned a map", ret);
}
+
+ private class ArtifactoryConfigLoaderMock extends ArtifactoryConfigLoader {
+ String getData;
+ HashMap<String, String> getDataMap = new HashMap<>();
+
+ public ArtifactoryConfigLoaderMock(Map<String, Object> conf) {
+ super(conf);
+ }
+
+ public void setData(String api, String artifact, String data) {
+ if (api == null) {
+ getData = data;
+ } else {
+ getDataMap.put(artifact, data);
+ }
+ }
+
+ @Override
+ protected String doGet(String api, String artifact, String host, Integer port) {
+ if (api == null) {
+ return getData;
+ }
+ return getDataMap.get(artifact);
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/scheduler/utils/FileConfigLoaderTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/utils/FileConfigLoaderTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/utils/FileConfigLoaderTest.java
index 2c711dc..8f1b4b8 100644
--- a/storm-server/src/test/java/org/apache/storm/scheduler/utils/FileConfigLoaderTest.java
+++ b/storm-server/src/test/java/org/apache/storm/scheduler/utils/FileConfigLoaderTest.java
@@ -1,34 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.utils;
+import java.io.File;
+import java.io.FileWriter;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
-import org.apache.storm.utils.Time;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.FileWriter;
-import java.util.HashMap;
-import java.util.Map;
import org.yaml.snakeyaml.Yaml;
public class FileConfigLoaderTest {
@@ -106,7 +99,7 @@ public class FileConfigLoaderTest {
Assert.assertEquals("Maps are a different size", testMap.keySet().size(), result.keySet().size());
- for (String key : testMap.keySet() ) {
+ for (String key : testMap.keySet()) {
Integer expectedValue = testMap.get(key);
Integer returnedValue = (Integer) result.get(key);
Assert.assertEquals("Bad value for key=" + key, expectedValue, returnedValue);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/security/auth/AuthTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/security/auth/AuthTest.java b/storm-server/src/test/java/org/apache/storm/security/auth/AuthTest.java
index 052f6cf..12cc009 100644
--- a/storm-server/src/test/java/org/apache/storm/security/auth/AuthTest.java
+++ b/storm-server/src/test/java/org/apache/storm/security/auth/AuthTest.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.security.auth;
@@ -38,7 +32,6 @@ import org.apache.storm.cluster.ClusterStateContext;
import org.apache.storm.cluster.ClusterUtils;
import org.apache.storm.cluster.DaemonType;
import org.apache.storm.cluster.IStormClusterState;
-import org.apache.storm.cluster.StormClusterStateImpl;
import org.apache.storm.generated.Nimbus;
import org.apache.storm.generated.WorkerToken;
import org.apache.storm.generated.WorkerTokenServiceType;
@@ -57,26 +50,28 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
public class AuthTest {
+ //3 seconds in milliseconds
+ public static final int NIMBUS_TIMEOUT = 3_000;
private static final Logger LOG = LoggerFactory.getLogger(AuthTest.class);
private static final File BASE = new File("./src/test/resources/");
-
- private static final String DIGEST_JAAS_CONF = new File(BASE,"jaas_digest.conf").getAbsolutePath();
+ private static final String DIGEST_JAAS_CONF = new File(BASE, "jaas_digest.conf").getAbsolutePath();
private static final String BAD_PASSWORD_CONF = new File(BASE, "jaas_digest_bad_password.conf").getAbsolutePath();
- private static final String WRONG_USER_CONF = new File(BASE,"jaas_digest_unknown_user.conf").getAbsolutePath();
+ private static final String WRONG_USER_CONF = new File(BASE, "jaas_digest_unknown_user.conf").getAbsolutePath();
private static final String MISSING_CLIENT = new File(BASE, "jaas_digest_missing_client.conf").getAbsolutePath();
- //3 seconds in milliseconds
- public static final int NIMBUS_TIMEOUT = 3_000;
-
- public interface MyBiConsumer<T, U> {
- void accept(T t, U u) throws Exception;
- }
-
-
public static Principal mkPrincipal(final String name) {
return new Principal() {
@Override
@@ -87,7 +82,7 @@ public class AuthTest {
@Override
public boolean equals(Object other) {
return other instanceof Principal
- && name.equals(((Principal) other).getName());
+ && name.equals(((Principal) other).getName());
}
@Override
@@ -104,7 +99,7 @@ public class AuthTest {
public static Subject mkSubject(String name) {
return new Subject(true, Collections.singleton(mkPrincipal(name)),
- Collections.emptySet(), Collections.emptySet());
+ Collections.emptySet(), Collections.emptySet());
}
public static void withServer(Class<? extends ITransportPlugin> transportPluginClass,
@@ -120,12 +115,12 @@ public class AuthTest {
withServer(loginCfg, transportPluginClass, impl, null, null, body);
}
- public static void withServer(String loginCfg,
- Class<? extends ITransportPlugin> transportPluginClass,
- Nimbus.Iface impl,
- InProcessZookeeper zk,
- Map<String, Object> extraConfs,
- MyBiConsumer<ThriftServer, Map<String, Object>> body) throws Exception {
+ public static void withServer(String loginCfg,
+ Class<? extends ITransportPlugin> transportPluginClass,
+ Nimbus.Iface impl,
+ InProcessZookeeper zk,
+ Map<String, Object> extraConfs,
+ MyBiConsumer<ThriftServer, Map<String, Object>> body) throws Exception {
Map<String, Object> conf = ConfigUtils.readStormConfig();
conf.put(Config.NIMBUS_THRIFT_PORT, 0);
conf.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, transportPluginClass.getName());
@@ -145,8 +140,8 @@ public class AuthTest {
Nimbus.Iface handler = impl != null ? impl : mock(Nimbus.Iface.class);
final ThriftServer server = new ThriftServer(conf,
- new Nimbus.Processor<>(handler),
- ThriftConnectionType.NIMBUS);
+ new Nimbus.Processor<>(handler),
+ ThriftConnectionType.NIMBUS);
LOG.info("Created Server... {}", server);
new Thread(() -> {
@@ -172,42 +167,6 @@ public class AuthTest {
}
}
- @Test
- public void kerbToLocalTest() {
- KerberosPrincipalToLocal kptol = new KerberosPrincipalToLocal();
- kptol.prepare(Collections.emptyMap());
- assertEquals("me", kptol.toLocal(mkPrincipal("me@realm")));
- assertEquals("simple", kptol.toLocal(mkPrincipal("simple")));
- assertEquals("someone", kptol.toLocal(mkPrincipal("someone/host@realm")));
- }
-
- @Test
- public void simpleAuthTest() throws Exception {
- Nimbus.Iface impl = mock(Nimbus.Iface.class);
- withServer(SimpleTransportPlugin.class,
- impl,
- (ThriftServer server, Map<String, Object> conf) -> {
- try (NimbusClient client = new NimbusClient(conf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) {
- client.getClient().activate("security_auth_test_topology");
- }
-
- //Verify digest is rejected...
- Map<String, Object> badConf = new HashMap<>(conf);
- badConf.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, DigestSaslTransportPlugin.class.getName());
- badConf.put("java.security.auth.login.config", DIGEST_JAAS_CONF);
- badConf.put(Config.STORM_NIMBUS_RETRY_TIMES, 0);
- try (NimbusClient client = new NimbusClient(badConf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) {
- client.getClient().activate("bad_security_auth_test_topology");
- fail("An exception should have been thrown trying to connect.");
- } catch (Exception te) {
- LOG.info("Got Exception...", te);
- assert(Utils.exceptionCauseIsInstanceOf(TTransportException.class, te));
- }
- });
- verify(impl).activate("security_auth_test_topology");
- verify(impl, never()).activate("bad_security_auth_test_topology");
- }
-
public static void verifyIncorrectJaasConf(ThriftServer server, Map<String, Object> conf, String jaas,
Class<? extends Exception> expectedException) {
Map<String, Object> badConf = new HashMap<>(conf);
@@ -217,55 +176,10 @@ public class AuthTest {
fail("An exception should have been thrown trying to connect.");
} catch (Exception e) {
LOG.info("Got Exception...", e);
- assert(Utils.exceptionCauseIsInstanceOf(expectedException, e));
+ assert (Utils.exceptionCauseIsInstanceOf(expectedException, e));
}
}
- @Test
- public void digestAuthTest() throws Exception {
- Nimbus.Iface impl = mock(Nimbus.Iface.class);
- final AtomicReference<ReqContext> user = new AtomicReference<>();
- doAnswer((invocation) -> {
- user.set(new ReqContext(ReqContext.context()));
- return null;
- }).when(impl).activate(anyString());
-
- withServer(DIGEST_JAAS_CONF,
- DigestSaslTransportPlugin.class,
- impl,
- (ThriftServer server, Map<String, Object> conf) -> {
- try (NimbusClient client = new NimbusClient(conf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) {
- client.getClient().activate("security_auth_test_topology");
- }
-
- conf.put(Config.STORM_NIMBUS_RETRY_TIMES, 0);
-
- //Verify simple is rejected...
- Map<String, Object> badTransport = new HashMap<>(conf);
- badTransport.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, SimpleTransportPlugin.class.getName());
- try (NimbusClient client = new NimbusClient(badTransport, "localhost", server.getPort(), NIMBUS_TIMEOUT)) {
- client.getClient().activate("bad_security_auth_test_topology");
- fail("An exception should have been thrown trying to connect.");
- } catch (Exception te) {
- LOG.info("Got Exception...", te);
- assert(Utils.exceptionCauseIsInstanceOf(TTransportException.class, te));
- }
- //The user here from the jaas conf is bob. No impersonation is done, so verify that
- ReqContext found = user.get();
- assertNotNull(found);
- assertEquals("bob", found.principal().getName());
- assertFalse(found.isImpersonating());
- user.set(null);
-
- verifyIncorrectJaasConf(server, conf, BAD_PASSWORD_CONF, TTransportException.class);
- verifyIncorrectJaasConf(server, conf, WRONG_USER_CONF, TTransportException.class);
- verifyIncorrectJaasConf(server, conf, "./nonexistent.conf", RuntimeException.class);
- verifyIncorrectJaasConf(server, conf, MISSING_CLIENT, IOException.class);
- });
- verify(impl).activate("security_auth_test_topology");
- verify(impl, never()).activate("bad_auth_test_topology");
- }
-
public static Subject createSubjectWith(WorkerToken wt) {
//This is a bit ugly, but it shows how this would happen in a worker so we will use the same APIs
Map<String, String> creds = new HashMap<>();
@@ -275,7 +189,7 @@ public class AuthTest {
return subject;
}
- public static void tryConnectAs( Map<String, Object> conf, ThriftServer server, Subject subject, String topoId)
+ public static void tryConnectAs(Map<String, Object> conf, ThriftServer server, Subject subject, String topoId)
throws PrivilegedActionException {
Subject.doAs(subject, (PrivilegedExceptionAction<Void>) () -> {
try (NimbusClient client = new NimbusClient(conf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) {
@@ -286,7 +200,7 @@ public class AuthTest {
}
public static Subject testConnectWithTokenFor(WorkerTokenManager wtMan, Map<String, Object> conf, ThriftServer server,
- String user, String topoId) throws PrivilegedActionException {
+ String user, String topoId) throws PrivilegedActionException {
WorkerToken wt = wtMan.createOrUpdateTokenFor(WorkerTokenServiceType.NIMBUS, user, topoId);
Subject subject = createSubjectWith(wt);
tryConnectAs(conf, server, subject, topoId);
@@ -302,6 +216,94 @@ public class AuthTest {
user.set(null);
}
+ public static ReqContext mkImpersonatingReqContext(String impersonatingUser, String userBeingIUmpersonated, InetAddress remoteAddress) {
+ ReqContext ret = new ReqContext(mkSubject(userBeingIUmpersonated));
+ ret.setRemoteAddress(remoteAddress);
+ ret.setRealPrincipal(mkPrincipal(impersonatingUser));
+ return ret;
+ }
+
+ @Test
+ public void kerbToLocalTest() {
+ KerberosPrincipalToLocal kptol = new KerberosPrincipalToLocal();
+ kptol.prepare(Collections.emptyMap());
+ assertEquals("me", kptol.toLocal(mkPrincipal("me@realm")));
+ assertEquals("simple", kptol.toLocal(mkPrincipal("simple")));
+ assertEquals("someone", kptol.toLocal(mkPrincipal("someone/host@realm")));
+ }
+
+ @Test
+ public void simpleAuthTest() throws Exception {
+ Nimbus.Iface impl = mock(Nimbus.Iface.class);
+ withServer(SimpleTransportPlugin.class,
+ impl,
+ (ThriftServer server, Map<String, Object> conf) -> {
+ try (NimbusClient client = new NimbusClient(conf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) {
+ client.getClient().activate("security_auth_test_topology");
+ }
+
+ //Verify digest is rejected...
+ Map<String, Object> badConf = new HashMap<>(conf);
+ badConf.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, DigestSaslTransportPlugin.class.getName());
+ badConf.put("java.security.auth.login.config", DIGEST_JAAS_CONF);
+ badConf.put(Config.STORM_NIMBUS_RETRY_TIMES, 0);
+ try (NimbusClient client = new NimbusClient(badConf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) {
+ client.getClient().activate("bad_security_auth_test_topology");
+ fail("An exception should have been thrown trying to connect.");
+ } catch (Exception te) {
+ LOG.info("Got Exception...", te);
+ assert (Utils.exceptionCauseIsInstanceOf(TTransportException.class, te));
+ }
+ });
+ verify(impl).activate("security_auth_test_topology");
+ verify(impl, never()).activate("bad_security_auth_test_topology");
+ }
+
+ @Test
+ public void digestAuthTest() throws Exception {
+ Nimbus.Iface impl = mock(Nimbus.Iface.class);
+ final AtomicReference<ReqContext> user = new AtomicReference<>();
+ doAnswer((invocation) -> {
+ user.set(new ReqContext(ReqContext.context()));
+ return null;
+ }).when(impl).activate(anyString());
+
+ withServer(DIGEST_JAAS_CONF,
+ DigestSaslTransportPlugin.class,
+ impl,
+ (ThriftServer server, Map<String, Object> conf) -> {
+ try (NimbusClient client = new NimbusClient(conf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) {
+ client.getClient().activate("security_auth_test_topology");
+ }
+
+ conf.put(Config.STORM_NIMBUS_RETRY_TIMES, 0);
+
+ //Verify simple is rejected...
+ Map<String, Object> badTransport = new HashMap<>(conf);
+ badTransport.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, SimpleTransportPlugin.class.getName());
+ try (NimbusClient client = new NimbusClient(badTransport, "localhost", server.getPort(), NIMBUS_TIMEOUT)) {
+ client.getClient().activate("bad_security_auth_test_topology");
+ fail("An exception should have been thrown trying to connect.");
+ } catch (Exception te) {
+ LOG.info("Got Exception...", te);
+ assert (Utils.exceptionCauseIsInstanceOf(TTransportException.class, te));
+ }
+ //The user here from the jaas conf is bob. No impersonation is done, so verify that
+ ReqContext found = user.get();
+ assertNotNull(found);
+ assertEquals("bob", found.principal().getName());
+ assertFalse(found.isImpersonating());
+ user.set(null);
+
+ verifyIncorrectJaasConf(server, conf, BAD_PASSWORD_CONF, TTransportException.class);
+ verifyIncorrectJaasConf(server, conf, WRONG_USER_CONF, TTransportException.class);
+ verifyIncorrectJaasConf(server, conf, "./nonexistent.conf", RuntimeException.class);
+ verifyIncorrectJaasConf(server, conf, MISSING_CLIENT, IOException.class);
+ });
+ verify(impl).activate("security_auth_test_topology");
+ verify(impl, never()).activate("bad_auth_test_topology");
+ }
+
@Test
public void workerTokenDigestAuthTest() throws Exception {
LOG.info("\n\n\t\tworkerTokenDigestAuthTest - START\n\n");
@@ -318,53 +320,54 @@ public class AuthTest {
try (InProcessZookeeper zk = new InProcessZookeeper()) {
withServer(MISSING_CLIENT,
- DigestSaslTransportPlugin.class,
- impl,
- zk,
- extraConfs,
- (ThriftServer server, Map<String, Object> conf) -> {
- try (Time.SimulatedTime sim = new Time.SimulatedTime()) {
- conf.put(Config.STORM_NIMBUS_RETRY_TIMES, 0);
- //We cannot connect if there is no client section in the jaas conf...
- try (NimbusClient client = new NimbusClient(conf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) {
- client.getClient().activate("bad_auth_test_topology");
- fail("We should not be able to connect without a token...");
- } catch (Exception e) {
- assert (Utils.exceptionCauseIsInstanceOf(IOException.class, e));
- }
-
- //Now lets create a token and verify that we can connect...
- IStormClusterState state = ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf));
- WorkerTokenManager wtMan = new WorkerTokenManager(conf, state);
- Subject bob = testConnectWithTokenFor(wtMan, conf, server, "bob", "topo-bob");
- verifyUserIs(user, "bob");
-
- Time.advanceTimeSecs(TimeUnit.HOURS.toSeconds(12));
-
- //Alice has no digest jaas section at all...
- Subject alice = testConnectWithTokenFor(wtMan, conf, server, "alice", "topo-alice");
- verifyUserIs(user, "alice");
-
- Time.advanceTimeSecs(TimeUnit.HOURS.toSeconds(13));
- //Verify that bob's token has expired
-
- try {
- tryConnectAs(conf, server, bob, "bad_auth_test_topology");
- fail("We should not be able to connect with bad auth");
- } catch (Exception e) {
- assert (Utils.exceptionCauseIsInstanceOf(TTransportException.class, e));
- }
- tryConnectAs(conf, server, alice, "topo-alice");
- verifyUserIs(user, "alice");
-
- //Now see if we can create a new token for bob and try again.
- bob = testConnectWithTokenFor(wtMan, conf, server, "bob", "topo-bob");
- verifyUserIs(user, "bob");
-
- tryConnectAs(conf, server, alice, "topo-alice");
- verifyUserIs(user, "alice");
- }
- });
+ DigestSaslTransportPlugin.class,
+ impl,
+ zk,
+ extraConfs,
+ (ThriftServer server, Map<String, Object> conf) -> {
+ try (Time.SimulatedTime sim = new Time.SimulatedTime()) {
+ conf.put(Config.STORM_NIMBUS_RETRY_TIMES, 0);
+ //We cannot connect if there is no client section in the jaas conf...
+ try (NimbusClient client = new NimbusClient(conf, "localhost", server.getPort(), NIMBUS_TIMEOUT)) {
+ client.getClient().activate("bad_auth_test_topology");
+ fail("We should not be able to connect without a token...");
+ } catch (Exception e) {
+ assert (Utils.exceptionCauseIsInstanceOf(IOException.class, e));
+ }
+
+ //Now lets create a token and verify that we can connect...
+ IStormClusterState state =
+ ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf));
+ WorkerTokenManager wtMan = new WorkerTokenManager(conf, state);
+ Subject bob = testConnectWithTokenFor(wtMan, conf, server, "bob", "topo-bob");
+ verifyUserIs(user, "bob");
+
+ Time.advanceTimeSecs(TimeUnit.HOURS.toSeconds(12));
+
+ //Alice has no digest jaas section at all...
+ Subject alice = testConnectWithTokenFor(wtMan, conf, server, "alice", "topo-alice");
+ verifyUserIs(user, "alice");
+
+ Time.advanceTimeSecs(TimeUnit.HOURS.toSeconds(13));
+ //Verify that bob's token has expired
+
+ try {
+ tryConnectAs(conf, server, bob, "bad_auth_test_topology");
+ fail("We should not be able to connect with bad auth");
+ } catch (Exception e) {
+ assert (Utils.exceptionCauseIsInstanceOf(TTransportException.class, e));
+ }
+ tryConnectAs(conf, server, alice, "topo-alice");
+ verifyUserIs(user, "alice");
+
+ //Now see if we can create a new token for bob and try again.
+ bob = testConnectWithTokenFor(wtMan, conf, server, "bob", "topo-bob");
+ verifyUserIs(user, "bob");
+
+ tryConnectAs(conf, server, alice, "topo-alice");
+ verifyUserIs(user, "alice");
+ }
+ });
}
verify(impl, times(2)).activate("topo-bob");
verify(impl, times(3)).activate("topo-alice");
@@ -585,13 +588,6 @@ public class AuthTest {
AuthUtils.getTransportPlugin(ThriftConnectionType.NIMBUS, conf, null);
}
- public static ReqContext mkImpersonatingReqContext(String impersonatingUser, String userBeingIUmpersonated, InetAddress remoteAddress) {
- ReqContext ret = new ReqContext(mkSubject(userBeingIUmpersonated));
- ret.setRemoteAddress(remoteAddress);
- ret.setRealPrincipal(mkPrincipal(impersonatingUser));
- return ret;
- }
-
@Test
public void impersonationAuthorizerTest() throws Exception {
final String impersonatingUser = "admin";
@@ -620,18 +616,22 @@ public class AuthTest {
//user with no impersonation acl should be reject
assertFalse(authorizer.permit(mkImpersonatingReqContext("user-with-no-acl", userBeingImpersonated, localHost),
- "someOperation",null));
+ "someOperation", null));
//request from hosts that are not authorized should be rejected
assertFalse(authorizer.permit(mkImpersonatingReqContext(impersonatingUser, userBeingImpersonated, unauthorizedHost),
- "someOperation", null));
+ "someOperation", null));
//request to impersonate users from unauthroized groups should be rejected.
assertFalse(authorizer.permit(mkImpersonatingReqContext(impersonatingUser, "unauthorized-user", localHost),
- "someOperation", null));
+ "someOperation", null));
//request from authorized hosts and group should be allowed.
assertTrue(authorizer.permit(mkImpersonatingReqContext(impersonatingUser, userBeingImpersonated, localHost),
- "someOperation", null));
+ "someOperation", null));
+ }
+
+ public interface MyBiConsumer<T, U> {
+ void accept(T t, U u) throws Exception;
}
}
\ No newline at end of file
[16/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopoCache.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopoCache.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopoCache.java
index 2b9de22..0b0e70c 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopoCache.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopoCache.java
@@ -1,32 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.daemon.nimbus;
-import static org.apache.storm.blobstore.BlobStoreAclHandler.READ;
-
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-
import javax.security.auth.Subject;
-
import org.apache.storm.blobstore.BlobStore;
import org.apache.storm.blobstore.BlobStoreAclHandler;
import org.apache.storm.generated.AccessControl;
@@ -41,6 +31,8 @@ import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.storm.blobstore.BlobStoreAclHandler.READ;
+
/**
* Cache topologies and topology confs from the blob store.
* Makes reading this faster because it can skip
@@ -48,22 +40,10 @@ import org.slf4j.LoggerFactory;
*/
public class TopoCache {
public static final Logger LOG = LoggerFactory.getLogger(TopoCache.class);
-
- private static final class WithAcl<T> {
- public final List<AccessControl> acl;
- public final T data;
-
- public WithAcl(List<AccessControl> acl, T data) {
- this.acl = acl;
- this.data = data;
- }
- }
-
private final BlobStore store;
private final BlobStoreAclHandler aclHandler;
private final ConcurrentHashMap<String, WithAcl<StormTopology>> topos = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, WithAcl<Map<String, Object>>> confs = new ConcurrentHashMap<>();
-
public TopoCache(BlobStore store, Map<String, Object> conf) {
this.store = store;
aclHandler = new BlobStoreAclHandler(conf);
@@ -243,4 +223,14 @@ public class TopoCache {
confs.clear();
topos.clear();
}
+
+ private static final class WithAcl<T> {
+ public final List<AccessControl> acl;
+ public final T data;
+
+ public WithAcl(List<AccessControl> acl, T data) {
+ this.acl = acl;
+ this.data = data;
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyActions.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyActions.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyActions.java
index 6a4a67a..bc41da5 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyActions.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyActions.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.nimbus;
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyResources.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyResources.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyResources.java
index 2812cce..bb932ed 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyResources.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyResources.java
@@ -1,26 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.daemon.nimbus;
import java.util.Collection;
import java.util.Map;
-
import org.apache.storm.generated.Assignment;
import org.apache.storm.generated.NodeInfo;
import org.apache.storm.generated.WorkerResources;
@@ -29,44 +22,6 @@ import org.apache.storm.scheduler.TopologyDetails;
import org.apache.storm.scheduler.WorkerSlot;
public final class TopologyResources {
- private static Collection<WorkerResources> getWorkerResources(SchedulerAssignment assignment) {
- Collection<WorkerResources> ret = null;
- if (assignment != null) {
- Map<WorkerSlot, WorkerResources> allResources = assignment.getScheduledResources();
- if (allResources != null) {
- ret = allResources.values();
- }
- }
- return ret;
- }
-
- private static Collection<WorkerResources> getWorkerResources(Assignment assignment) {
- Collection<WorkerResources> ret = null;
- if (assignment != null) {
- Map<NodeInfo, WorkerResources> allResources = assignment.get_worker_resources();
- if (allResources != null) {
- ret = allResources.values();
- }
- }
- return ret;
- }
-
- private static Map<String, Double> getNodeIdToSharedOffHeap(SchedulerAssignment assignment) {
- Map<String, Double> ret = null;
- if (assignment != null) {
- ret = assignment.getNodeIdToTotalSharedOffHeapMemory();
- }
- return ret;
- }
-
- private static Map<String, Double> getNodeIdToSharedOffHeap(Assignment assignment) {
- Map<String, Double> ret = null;
- if (assignment != null) {
- ret = assignment.get_total_shared_off_heap();
- }
- return ret;
- }
-
private final double requestedMemOnHeap;
private final double requestedMemOffHeap;
private final double requestedSharedMemOnHeap;
@@ -81,7 +36,6 @@ public final class TopologyResources {
private double assignedNonSharedMemOnHeap;
private double assignedNonSharedMemOffHeap;
private double assignedCpu;
-
private TopologyResources(TopologyDetails td, Collection<WorkerResources> workers,
Map<String, Double> sharedOffHeap) {
requestedMemOnHeap = td.getTotalRequestedMemOnHeap();
@@ -125,15 +79,12 @@ public final class TopologyResources {
assignedMemOffHeap += sharedOff;
}
}
-
public TopologyResources(TopologyDetails td, SchedulerAssignment assignment) {
this(td, getWorkerResources(assignment), getNodeIdToSharedOffHeap(assignment));
}
-
public TopologyResources(TopologyDetails td, Assignment assignment) {
this(td, getWorkerResources(assignment), getNodeIdToSharedOffHeap(assignment));
}
-
public TopologyResources() {
this(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
}
@@ -169,6 +120,44 @@ public final class TopologyResources {
this.assignedCpu = assignedCpu;
}
+ private static Collection<WorkerResources> getWorkerResources(SchedulerAssignment assignment) {
+ Collection<WorkerResources> ret = null;
+ if (assignment != null) {
+ Map<WorkerSlot, WorkerResources> allResources = assignment.getScheduledResources();
+ if (allResources != null) {
+ ret = allResources.values();
+ }
+ }
+ return ret;
+ }
+
+ private static Collection<WorkerResources> getWorkerResources(Assignment assignment) {
+ Collection<WorkerResources> ret = null;
+ if (assignment != null) {
+ Map<NodeInfo, WorkerResources> allResources = assignment.get_worker_resources();
+ if (allResources != null) {
+ ret = allResources.values();
+ }
+ }
+ return ret;
+ }
+
+ private static Map<String, Double> getNodeIdToSharedOffHeap(SchedulerAssignment assignment) {
+ Map<String, Double> ret = null;
+ if (assignment != null) {
+ ret = assignment.getNodeIdToTotalSharedOffHeapMemory();
+ }
+ return ret;
+ }
+
+ private static Map<String, Double> getNodeIdToSharedOffHeap(Assignment assignment) {
+ Map<String, Double> ret = null;
+ if (assignment != null) {
+ ret = assignment.get_total_shared_off_heap();
+ }
+ return ret;
+ }
+
public double getRequestedMemOnHeap() {
return requestedMemOnHeap;
}
@@ -209,6 +198,10 @@ public final class TopologyResources {
return assignedSharedMemOnHeap;
}
+ public void setAssignedSharedMemOnHeap(double assignedSharedMemOnHeap) {
+ this.assignedSharedMemOnHeap = assignedSharedMemOnHeap;
+ }
+
public double getRequestedSharedMemOnHeap() {
return requestedSharedMemOnHeap;
}
@@ -225,10 +218,6 @@ public final class TopologyResources {
return requestedNonSharedMemOffHeap;
}
- public void setAssignedSharedMemOnHeap(double assignedSharedMemOnHeap) {
- this.assignedSharedMemOnHeap = assignedSharedMemOnHeap;
- }
-
public double getAssignedSharedMemOffHeap() {
return assignedSharedMemOffHeap;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyStateTransition.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyStateTransition.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyStateTransition.java
index 39151e4..37019b9 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyStateTransition.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/TopologyStateTransition.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.nimbus;
import org.apache.storm.generated.StormBase;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
index e2ecbfb..2358f2a 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -8,7 +8,7 @@
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,12 +18,8 @@
package org.apache.storm.daemon.supervisor;
-import static org.apache.storm.daemon.nimbus.Nimbus.MIN_VERSION_SUPPORT_RPC_HEARTBEAT;
-import static org.apache.storm.utils.Utils.OR;
-
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
-
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
@@ -38,7 +34,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NavigableMap;
-
import org.apache.commons.lang.StringUtils;
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
@@ -60,84 +55,75 @@ import org.apache.storm.utils.VersionInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.storm.daemon.nimbus.Nimbus.MIN_VERSION_SUPPORT_RPC_HEARTBEAT;
+import static org.apache.storm.utils.Utils.OR;
+
/**
* A container that runs processes on the local box.
*/
public class BasicContainer extends Container {
+ static final TopoMetaLRUCache TOPO_META_CACHE = new TopoMetaLRUCache();
private static final Logger LOG = LoggerFactory.getLogger(BasicContainer.class);
private static final FilenameFilter jarFilter = (dir, name) -> name.endsWith(".jar");
- private static final Joiner CPJ =
- Joiner.on(File.pathSeparator).skipNulls();
-
+ private static final Joiner CPJ =
+ Joiner.on(File.pathSeparator).skipNulls();
protected final LocalState _localState;
protected final String _profileCmd;
protected final String _stormHome = System.getProperty("storm.home");
- protected volatile boolean _exitedEarly = false;
- protected volatile long memoryLimitMB;
- protected volatile long memoryLimitExceededStart;
protected final double hardMemoryLimitMultiplier;
protected final long hardMemoryLimitOver;
protected final long lowMemoryThresholdMB;
protected final long mediumMemoryThresholdMb;
protected final long mediumMemoryGracePeriodMs;
-
- private class ProcessExitCallback implements ExitCodeCallback {
- private final String _logPrefix;
-
- public ProcessExitCallback(String logPrefix) {
- _logPrefix = logPrefix;
- }
-
- @Override
- public void call(int exitCode) {
- LOG.info("{} exited with code: {}", _logPrefix, exitCode);
- _exitedEarly = true;
- }
- }
+ protected volatile boolean _exitedEarly = false;
+ protected volatile long memoryLimitMB;
+ protected volatile long memoryLimitExceededStart;
/**
* Create a new BasicContainer.
- * @param type the type of container being made.
- * @param conf the supervisor config
- * @param supervisorId the ID of the supervisor this is a part of.
- * @param supervisorPort the thrift server port of the supervisor this is a part of.
- * @param port the port the container is on. Should be <= 0 if only a partial recovery
- * @param assignment the assignment for this container. Should be null if only a partial recovery.
+ *
+ * @param type the type of container being made.
+ * @param conf the supervisor config
+ * @param supervisorId the ID of the supervisor this is a part of.
+ * @param supervisorPort the thrift server port of the supervisor this is a part of.
+ * @param port the port the container is on. Should be <= 0 if only a partial recovery
+ * @param assignment the assignment for this container. Should be null if only a partial recovery.
* @param resourceIsolationManager used to isolate resources for a container can be null if no isolation is used.
- * @param localState the local state of the supervisor. May be null if partial recovery
- * @param workerId the id of the worker to use. Must not be null if doing a partial recovery.
+ * @param localState the local state of the supervisor. May be null if partial recovery
+ * @param workerId the id of the worker to use. Must not be null if doing a partial recovery.
*/
public BasicContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int supervisorPort,
- int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager,
- LocalState localState, String workerId) throws IOException {
+ int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager,
+ LocalState localState, String workerId) throws IOException {
this(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, localState,
- workerId, null, null, null);
+ workerId, null, null, null);
}
/**
* Create a new BasicContainer.
- * @param type the type of container being made.
- * @param conf the supervisor config
- * @param supervisorId the ID of the supervisor this is a part of.
- * @param supervisorPort the thrift server port of the supervisor this is a part of.
- * @param port the port the container is on. Should be <= 0 if only a partial recovery
- * @param assignment the assignment for this container. Should be null if only a partial recovery.
+ *
+ * @param type the type of container being made.
+ * @param conf the supervisor config
+ * @param supervisorId the ID of the supervisor this is a part of.
+ * @param supervisorPort the thrift server port of the supervisor this is a part of.
+ * @param port the port the container is on. Should be <= 0 if only a partial recovery
+ * @param assignment the assignment for this container. Should be null if only a partial recovery.
* @param resourceIsolationManager used to isolate resources for a container can be null if no isolation is used.
- * @param localState the local state of the supervisor. May be null if partial recovery
- * @param workerId the id of the worker to use. Must not be null if doing a partial recovery.
- * @param ops file system operations (mostly for testing) if null a new one is made
- * @param topoConf the config of the topology (mostly for testing) if null
- * and not a partial recovery the real conf is read.
- * @param profileCmd the command to use when profiling (used for testing)
- * @throws IOException on any error
+ * @param localState the local state of the supervisor. May be null if partial recovery
+ * @param workerId the id of the worker to use. Must not be null if doing a partial recovery.
+ * @param ops file system operations (mostly for testing) if null a new one is made
+ * @param topoConf the config of the topology (mostly for testing) if null and not a partial recovery the real conf is
+ * read.
+ * @param profileCmd the command to use when profiling (used for testing)
+ * @throws IOException on any error
* @throws ContainerRecoveryException if the Container could not be recovered.
*/
BasicContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int supervisorPort, int port,
- LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager,
- LocalState localState, String workerId, Map<String, Object> topoConf,
- AdvancedFSOps ops, String profileCmd) throws IOException {
+ LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager,
+ LocalState localState, String workerId, Map<String, Object> topoConf,
+ AdvancedFSOps ops, String profileCmd) throws IOException {
super(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, workerId, topoConf, ops);
- assert(localState != null);
+ assert (localState != null);
_localState = localState;
if (type.isRecovery() && !type.isOnlyKillable()) {
@@ -161,7 +147,7 @@ public class BasicContainer extends Container {
if (profileCmd == null) {
profileCmd = _stormHome + File.separator + "bin" + File.separator
- + conf.get(DaemonConfig.WORKER_PROFILER_COMMAND);
+ + conf.get(DaemonConfig.WORKER_PROFILER_COMMAND);
}
_profileCmd = profileCmd;
@@ -181,14 +167,33 @@ public class BasicContainer extends Container {
}
}
+ private static void removeWorkersOn(Map<String, Integer> workerToPort, int _port) {
+ for (Iterator<Entry<String, Integer>> i = workerToPort.entrySet().iterator(); i.hasNext(); ) {
+ Entry<String, Integer> found = i.next();
+ if (_port == found.getValue().intValue()) {
+ LOG.warn("Deleting worker {} from state", found.getKey());
+ i.remove();
+ }
+ }
+ }
+
+ public static List<String> getDependencyLocationsFor(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops,
+ String stormRoot) throws IOException {
+ return TOPO_META_CACHE.get(conf, topologyId, ops, stormRoot).getDepLocs();
+ }
+
+ public static String getStormVersionFor(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops,
+ String stormRoot) throws IOException {
+ return TOPO_META_CACHE.get(conf, topologyId, ops, stormRoot).getStormVersion();
+ }
+
/**
- * Create a new worker ID for this process and store in in this object and
- * in the local state. Never call this if a worker is currently up and running.
- * We will lose track of the process.
+ * Create a new worker ID for this process and store in in this object and in the local state. Never call this if a worker is currently
+ * up and running. We will lose track of the process.
*/
protected void createNewWorkerId() {
_type.assertFull();
- assert(_workerId == null);
+ assert (_workerId == null);
synchronized (_localState) {
_workerId = Utils.uuid();
Map<String, Integer> workerToPort = _localState.getApprovedWorkers();
@@ -202,16 +207,6 @@ public class BasicContainer extends Container {
}
}
- private static void removeWorkersOn(Map<String, Integer> workerToPort, int _port) {
- for (Iterator<Entry<String, Integer>> i = workerToPort.entrySet().iterator(); i.hasNext();) {
- Entry<String, Integer> found = i.next();
- if (_port == found.getValue().intValue()) {
- LOG.warn("Deleting worker {} from state", found.getKey());
- i.remove();
- }
- }
- }
-
@Override
public void cleanUpForRestart() throws IOException {
String origWorkerId = _workerId;
@@ -243,22 +238,17 @@ public class BasicContainer extends Container {
/**
* Run the given command for profiling.
*
- * @param command
- * the command to run
- * @param env
- * the environment to run the command
- * @param logPrefix
- * the prefix to include in the logs
- * @param targetDir
- * the working directory to run the command in
+ * @param command the command to run
+ * @param env the environment to run the command
+ * @param logPrefix the prefix to include in the logs
+ * @param targetDir the working directory to run the command in
* @return true if it ran successfully, else false
- * @throws IOException
- * on any error
- * @throws InterruptedException
- * if interrupted wile waiting for the process to exit.
+ *
+ * @throws IOException on any error
+ * @throws InterruptedException if interrupted wile waiting for the process to exit.
*/
protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix,
- File targetDir) throws IOException, InterruptedException {
+ File targetDir) throws IOException, InterruptedException {
_type.assertFull();
Process p = ClientSupervisorUtils.launchProcess(command, env, logPrefix, null, targetDir);
int ret = p.waitFor();
@@ -282,7 +272,7 @@ public class BasicContainer extends Container {
ProfileAction profileAction = request.get_action();
String logPrefix = "ProfilerAction process " + _topologyId + ":" + _port + " PROFILER_ACTION: " + profileAction
- + " ";
+ + " ";
List<String> command = mkProfileCommand(profileAction, stop, workerPid, targetDir);
@@ -296,14 +286,15 @@ public class BasicContainer extends Container {
/**
* Get the command to run when doing profiling.
- * @param action the profiling action to perform
- * @param stop if this is meant to stop the profiling or start it
+ *
+ * @param action the profiling action to perform
+ * @param stop if this is meant to stop the profiling or start it
* @param workerPid the PID of the process to profile
* @param targetDir the current working directory of the worker process
* @return the command to run for profiling.
*/
private List<String> mkProfileCommand(ProfileAction action, boolean stop, String workerPid, String targetDir) {
- switch(action) {
+ switch (action) {
case JMAP_DUMP:
return jmapDumpCmd(workerPid, targetDir);
case JSTACK_DUMP:
@@ -347,10 +338,11 @@ public class BasicContainer extends Container {
}
/**
- * Compute the java.library.path that should be used for the worker.
- * This helps it to load JNI libraries that are packaged in the uber jar.
+ * Compute the java.library.path that should be used for the worker. This helps it to load JNI libraries that are packaged in the uber
+ * jar.
+ *
* @param stormRoot the root directory of the worker process
- * @param conf the config for the supervisor.
+ * @param conf the config for the supervisor.
* @return the java.library.path/LD_LIBRARY_PATH to use so native libraries load correctly.
*/
protected String javaLibraryPath(String stormRoot, Map<String, Object> conf) {
@@ -359,13 +351,13 @@ public class BasicContainer extends Container {
String arch = System.getProperty("os.arch");
String archResourceRoot = resourceRoot + File.separator + os + "-" + arch;
String ret = CPJ.join(archResourceRoot, resourceRoot,
- conf.get(DaemonConfig.JAVA_LIBRARY_PATH));
+ conf.get(DaemonConfig.JAVA_LIBRARY_PATH));
return ret;
}
/**
- * Returns a path with a wildcard as the final element, so that the JVM will expand
- * that to all JARs in the directory.
+ * Returns a path with a wildcard as the final element, so that the JVM will expand that to all JARs in the directory.
+ *
* @param dir the directory to which a wildcard will be appended
* @return the path with wildcard ("*") suffix
*/
@@ -376,7 +368,7 @@ public class BasicContainer extends Container {
protected List<String> frameworkClasspath(SimpleVersion topoVersion) {
File stormWorkerLibDir = new File(_stormHome, "lib-worker");
String topoConfDir =
- System.getenv("STORM_CONF_DIR") != null ?
+ System.getenv("STORM_CONF_DIR") != null ?
System.getenv("STORM_CONF_DIR") :
new File(_stormHome, "conf").getAbsolutePath();
File stormExtlibDir = new File(_stormHome, "extlib");
@@ -401,7 +393,7 @@ public class BasicContainer extends Container {
//Have not moved to a java worker yet
defaultWorkerGuess = "org.apache.storm.daemon.worker";
}
- NavigableMap<SimpleVersion,String> mains = Utils.getConfiguredWorkerMainVersions(_conf);
+ NavigableMap<SimpleVersion, String> mains = Utils.getConfiguredWorkerMainVersions(_conf);
return Utils.getCompatibleVersion(mains, topoVersion, "worker main class", defaultWorkerGuess);
}
@@ -411,25 +403,26 @@ public class BasicContainer extends Container {
//Prior to the org.apache change
defaultGuess = "backtype.storm.LogWriter";
}
- NavigableMap<SimpleVersion,String> mains = Utils.getConfiguredWorkerLogWriterVersions(_conf);
+ NavigableMap<SimpleVersion, String> mains = Utils.getConfiguredWorkerLogWriterVersions(_conf);
return Utils.getCompatibleVersion(mains, topoVersion, "worker log writer class", defaultGuess);
}
@SuppressWarnings("unchecked")
private List<String> asStringList(Object o) {
if (o instanceof String) {
- return Arrays.asList((String)o);
+ return Arrays.asList((String) o);
} else if (o instanceof List) {
- return (List<String>)o;
+ return (List<String>) o;
}
return Collections.EMPTY_LIST;
}
/**
* Compute the classpath for the worker process.
- * @param stormJar the topology jar
+ *
+ * @param stormJar the topology jar
* @param dependencyLocations any dependencies from the topology
- * @param topoVersion the version of the storm framework to use
+ * @param topoVersion the version of the storm framework to use
* @return the full classpath
*/
protected String getWorkerClassPath(String stormJar, List<String> dependencyLocations, SimpleVersion topoVersion) {
@@ -469,7 +462,7 @@ public class BasicContainer extends Container {
String string = substituteChildOptsInternal((String) value, memOnheap);
if (StringUtils.isNotBlank(string)) {
String[] strings = string.split("\\s+");
- for (String s: strings) {
+ for (String s : strings) {
if (StringUtils.isNotBlank(s)) {
rets.add(s);
}
@@ -491,24 +484,19 @@ public class BasicContainer extends Container {
/**
* Launch the worker process (non-blocking).
*
- * @param command
- * the command to run
- * @param env
- * the environment to run the command
- * @param processExitCallback
- * a callback for when the process exits
- * @param logPrefix
- * the prefix to include in the logs
- * @param targetDir
- * the working directory to run the command in
+ * @param command the command to run
+ * @param env the environment to run the command
+ * @param processExitCallback a callback for when the process exits
+ * @param logPrefix the prefix to include in the logs
+ * @param targetDir the working directory to run the command in
* @return true if it ran successfully, else false
- * @throws IOException
- * on any error
+ *
+ * @throws IOException on any error
*/
protected void launchWorkerProcess(List<String> command, Map<String, String> env, String logPrefix,
- ExitCodeCallback processExitCallback, File targetDir) throws IOException {
+ ExitCodeCallback processExitCallback, File targetDir) throws IOException {
if (_resourceIsolationManager != null) {
- command = _resourceIsolationManager.getLaunchCommand(_workerId, command);
+ command = _resourceIsolationManager.getLaunchCommand(_workerId, command);
}
ClientSupervisorUtils.launchProcess(command, env, logPrefix, processExitCallback, targetDir);
}
@@ -530,107 +518,12 @@ public class BasicContainer extends Container {
return log4jConfigurationDir + File.separator + "worker.xml";
}
- private static class TopologyMetaData {
- private boolean _dataCached = false;
- private List<String> _depLocs = null;
- private String _stormVersion = null;
- private final Map<String, Object> _conf;
- private final String _topologyId;
- private final AdvancedFSOps _ops;
- private final String _stormRoot;
-
- public TopologyMetaData(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, final String stormRoot) {
- _conf = conf;
- _topologyId = topologyId;
- _ops = ops;
- _stormRoot = stormRoot;
- }
-
- public String toString() {
- List<String> data;
- String stormVersion;
- synchronized(this) {
- data = _depLocs;
- stormVersion = _stormVersion;
- }
- return "META for " + _topologyId +" DEP_LOCS => " + data + " STORM_VERSION => " + stormVersion;
- }
-
- private synchronized void readData() throws IOException {
- final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _ops);
- final List<String> dependencyLocations = new ArrayList<>();
- if (stormTopology.get_dependency_jars() != null) {
- for (String dependency : stormTopology.get_dependency_jars()) {
- dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath());
- }
- }
-
- if (stormTopology.get_dependency_artifacts() != null) {
- for (String dependency : stormTopology.get_dependency_artifacts()) {
- dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath());
- }
- }
- _depLocs = dependencyLocations;
- _stormVersion = stormTopology.get_storm_version();
- _dataCached = true;
- }
-
- public synchronized List<String> getDepLocs() throws IOException {
- if (!_dataCached) {
- readData();
- }
- return _depLocs;
- }
-
- public synchronized String getStormVersion() throws IOException {
- if (!_dataCached) {
- readData();
- }
- return _stormVersion;
- }
- }
-
- static class TopoMetaLRUCache {
- public final int _maxSize = 100; //We could make this configurable in the future...
-
- @SuppressWarnings("serial")
- private LinkedHashMap<String, TopologyMetaData> _cache = new LinkedHashMap<String, TopologyMetaData>() {
- @Override
- protected boolean removeEldestEntry(Map.Entry<String,TopologyMetaData> eldest) {
- return (size() > _maxSize);
- }
- };
-
- public synchronized TopologyMetaData get(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, String stormRoot) {
- //Only go off of the topology id for now.
- TopologyMetaData dl = _cache.get(topologyId);
- if (dl == null) {
- _cache.putIfAbsent(topologyId, new TopologyMetaData(conf, topologyId, ops, stormRoot));
- dl = _cache.get(topologyId);
- }
- return dl;
- }
-
- public synchronized void clear() {
- _cache.clear();
- }
- }
-
- static final TopoMetaLRUCache TOPO_META_CACHE = new TopoMetaLRUCache();
-
- public static List<String> getDependencyLocationsFor(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, String stormRoot) throws IOException {
- return TOPO_META_CACHE.get(conf, topologyId, ops, stormRoot).getDepLocs();
- }
-
- public static String getStormVersionFor(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, String stormRoot) throws IOException {
- return TOPO_META_CACHE.get(conf, topologyId, ops, stormRoot).getStormVersion();
- }
-
/**
- * Get parameters for the class path of the worker process. Also used by the
- * log Writer.
+ * Get parameters for the class path of the worker process. Also used by the log Writer.
+ *
* @param stormRoot the root dist dir for the topology
* @return the classpath for the topology as command line arguments.
+ *
* @throws IOException on any error.
*/
private List<String> getClassPathParams(final String stormRoot, final SimpleVersion topoVersion) throws IOException {
@@ -645,8 +538,9 @@ public class BasicContainer extends Container {
}
/**
- * Get a set of java properties that are common to both the log writer and the worker processes.
- * These are mostly system properties that are used by logging.
+ * Get a set of java properties that are common to both the log writer and the worker processes. These are mostly system properties that
+ * are used by logging.
+ *
* @return a list of command line options
*/
private List<String> getCommonParams() {
@@ -667,7 +561,7 @@ public class BasicContainer extends Container {
commonParams.add("-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector");
commonParams.add("-Dstorm.local.dir=" + _conf.get(Config.STORM_LOCAL_DIR));
if (memoryLimitMB > 0) {
- commonParams.add("-Dworker.memory_limit_mb="+ memoryLimitMB);
+ commonParams.add("-Dworker.memory_limit_mb=" + memoryLimitMB);
}
return commonParams;
}
@@ -675,7 +569,7 @@ public class BasicContainer extends Container {
private int getMemOnHeap(WorkerResources resources) {
int memOnheap = 0;
if (resources != null && resources.is_set_mem_on_heap() &&
- resources.get_mem_on_heap() > 0) {
+ resources.get_mem_on_heap() > 0) {
memOnheap = (int) Math.ceil(resources.get_mem_on_heap());
} else {
// set the default heap memory size for supervisor-test
@@ -705,21 +599,23 @@ public class BasicContainer extends Container {
/**
* Create the command to launch the worker process.
+ *
* @param memOnheap the on heap memory for the worker
* @param stormRoot the root dist dir for the topology
- * @param jlp java library path for the topology
+ * @param jlp java library path for the topology
* @return the command to run
+ *
* @throws IOException on any error.
*/
private List<String> mkLaunchCommand(final int memOnheap, final String stormRoot,
- final String jlp) throws IOException {
+ final String jlp) throws IOException {
final String javaCmd = javaCmd("java");
final String stormOptions = ConfigUtils.concatIfNotNull(System.getProperty("storm.options"));
final String topoConfFile = ConfigUtils.concatIfNotNull(System.getProperty("storm.conf.file"));
final String workerTmpDir = ConfigUtils.workerTmpRoot(_conf, _workerId);
String topoVersionString = getStormVersionFor(_conf, _topologyId, _ops, stormRoot);
if (topoVersionString == null) {
- topoVersionString = (String)_conf.getOrDefault(Config.SUPERVISOR_WORKER_DEFAULT_VERSION, VersionInfo.getVersion());
+ topoVersionString = (String) _conf.getOrDefault(Config.SUPERVISOR_WORKER_DEFAULT_VERSION, VersionInfo.getVersion());
}
final SimpleVersion topoVersion = new SimpleVersion(topoVersionString);
@@ -744,8 +640,8 @@ public class BasicContainer extends Container {
commandList.addAll(substituteChildopts(_conf.get(Config.WORKER_CHILDOPTS), memOnheap));
commandList.addAll(substituteChildopts(_topoConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), memOnheap));
commandList.addAll(substituteChildopts(Utils.OR(
- _topoConf.get(Config.TOPOLOGY_WORKER_GC_CHILDOPTS),
- _conf.get(Config.WORKER_GC_CHILDOPTS)), memOnheap));
+ _topoConf.get(Config.TOPOLOGY_WORKER_GC_CHILDOPTS),
+ _conf.get(Config.WORKER_GC_CHILDOPTS)), memOnheap));
commandList.addAll(getWorkerProfilerChildOpts(memOnheap));
commandList.add("-Djava.library.path=" + jlp);
commandList.add("-Dstorm.conf.file=" + topoConfFile);
@@ -759,112 +655,112 @@ public class BasicContainer extends Container {
// supervisor port should be only presented to worker which supports RPC heartbeat
// unknown version should be treated as "current version", which supports RPC heartbeat
if ((topoVersion.getMajor() == -1 && topoVersion.getMinor() == -1) ||
- topoVersion.compareTo(MIN_VERSION_SUPPORT_RPC_HEARTBEAT) >= 0) {
+ topoVersion.compareTo(MIN_VERSION_SUPPORT_RPC_HEARTBEAT) >= 0) {
commandList.add(String.valueOf(_supervisorPort));
}
commandList.add(String.valueOf(_port));
commandList.add(_workerId);
-
+
return commandList;
}
-
- @Override
- public boolean isMemoryLimitViolated(LocalAssignment withUpdatedLimits) throws IOException {
- if (super.isMemoryLimitViolated(withUpdatedLimits)) {
- return true;
- }
- if (_resourceIsolationManager != null) {
- // In the short term the goal is to not shoot anyone unless we really need to.
- // The on heap should limit the memory usage in most cases to a reasonable amount
- // If someone is using way more than they requested this is a bug and we should
- // not allow it
- long usageMb;
- long memoryLimitMb;
- long hardMemoryLimitOver;
- String typeOfCheck;
-
- if (withUpdatedLimits.is_set_total_node_shared()) {
- //We need to do enforcement on a topology level, not a single worker level...
- // Because in for cgroups each page in shared memory goes to the worker that touched it
- // first. We may need to make this more plugable in the future and let the resource
- // isolation manager tell us what to do
- usageMb = getTotalTopologyMemoryUsed();
- memoryLimitMb = getTotalTopologyMemoryReserved(withUpdatedLimits);
- hardMemoryLimitOver = this.hardMemoryLimitOver * getTotalWorkersForThisTopology();
- typeOfCheck = "TOPOLOGY " + _topologyId;
- } else {
- usageMb = getMemoryUsageMb();
- memoryLimitMb = this.memoryLimitMB;
- hardMemoryLimitOver = this.hardMemoryLimitOver;
- typeOfCheck = "WORKER " + _workerId;
- }
- LOG.debug(
- "Enforcing memory usage for {} with usage of {} out of {} total and a hard limit of {}",
- typeOfCheck,
- usageMb,
- memoryLimitMb,
- hardMemoryLimitOver);
-
- if (usageMb <= 0) {
- //Looks like usage might not be supported
- return false;
- }
- long hardLimitMb = Math.max((long)(memoryLimitMb * hardMemoryLimitMultiplier), memoryLimitMb + hardMemoryLimitOver);
- if (usageMb > hardLimitMb) {
- LOG.warn(
- "{} is using {} MB > adjusted hard limit {} MB", typeOfCheck, usageMb, hardLimitMb);
- return true;
- }
- if (usageMb > memoryLimitMb) {
- //For others using too much it is really a question of how much memory is free in the system
- // to be use. If we cannot calculate it assume that it is bad
- long systemFreeMemoryMb = 0;
- try {
- systemFreeMemoryMb = _resourceIsolationManager.getSystemFreeMemoryMb();
- } catch (IOException e) {
- LOG.warn("Error trying to calculate free memory on the system {}", e);
- }
- LOG.debug("SYSTEM MEMORY FREE {} MB", systemFreeMemoryMb);
- //If the system is low on memory we cannot be kind and need to shoot something
- if (systemFreeMemoryMb <= lowMemoryThresholdMB) {
- LOG.warn(
- "{} is using {} MB > memory limit {} MB and system is low on memory {} free",
- typeOfCheck,
- usageMb,
- memoryLimitMb,
- systemFreeMemoryMb);
- return true;
- }
-
- //If the system still has some free memory give them a grace period to
- // drop back down.
- if (systemFreeMemoryMb < mediumMemoryThresholdMb) {
- if (memoryLimitExceededStart < 0) {
- memoryLimitExceededStart = Time.currentTimeMillis();
- } else {
- long timeInViolation = Time.currentTimeMillis() - memoryLimitExceededStart;
- if (timeInViolation > mediumMemoryGracePeriodMs) {
- LOG.warn(
- "{} is using {} MB > memory limit {} MB for {} seconds",
- typeOfCheck,
- usageMb,
- memoryLimitMb,
- timeInViolation / 1000);
- return true;
+
+ @Override
+ public boolean isMemoryLimitViolated(LocalAssignment withUpdatedLimits) throws IOException {
+ if (super.isMemoryLimitViolated(withUpdatedLimits)) {
+ return true;
+ }
+ if (_resourceIsolationManager != null) {
+ // In the short term the goal is to not shoot anyone unless we really need to.
+ // The on heap should limit the memory usage in most cases to a reasonable amount
+ // If someone is using way more than they requested this is a bug and we should
+ // not allow it
+ long usageMb;
+ long memoryLimitMb;
+ long hardMemoryLimitOver;
+ String typeOfCheck;
+
+ if (withUpdatedLimits.is_set_total_node_shared()) {
+ //We need to do enforcement on a topology level, not a single worker level...
+ // Because in for cgroups each page in shared memory goes to the worker that touched it
+ // first. We may need to make this more plugable in the future and let the resource
+ // isolation manager tell us what to do
+ usageMb = getTotalTopologyMemoryUsed();
+ memoryLimitMb = getTotalTopologyMemoryReserved(withUpdatedLimits);
+ hardMemoryLimitOver = this.hardMemoryLimitOver * getTotalWorkersForThisTopology();
+ typeOfCheck = "TOPOLOGY " + _topologyId;
+ } else {
+ usageMb = getMemoryUsageMb();
+ memoryLimitMb = this.memoryLimitMB;
+ hardMemoryLimitOver = this.hardMemoryLimitOver;
+ typeOfCheck = "WORKER " + _workerId;
+ }
+ LOG.debug(
+ "Enforcing memory usage for {} with usage of {} out of {} total and a hard limit of {}",
+ typeOfCheck,
+ usageMb,
+ memoryLimitMb,
+ hardMemoryLimitOver);
+
+ if (usageMb <= 0) {
+ //Looks like usage might not be supported
+ return false;
+ }
+ long hardLimitMb = Math.max((long) (memoryLimitMb * hardMemoryLimitMultiplier), memoryLimitMb + hardMemoryLimitOver);
+ if (usageMb > hardLimitMb) {
+ LOG.warn(
+ "{} is using {} MB > adjusted hard limit {} MB", typeOfCheck, usageMb, hardLimitMb);
+ return true;
+ }
+ if (usageMb > memoryLimitMb) {
+ //For others using too much it is really a question of how much memory is free in the system
+ // to be use. If we cannot calculate it assume that it is bad
+ long systemFreeMemoryMb = 0;
+ try {
+ systemFreeMemoryMb = _resourceIsolationManager.getSystemFreeMemoryMb();
+ } catch (IOException e) {
+ LOG.warn("Error trying to calculate free memory on the system {}", e);
+ }
+ LOG.debug("SYSTEM MEMORY FREE {} MB", systemFreeMemoryMb);
+ //If the system is low on memory we cannot be kind and need to shoot something
+ if (systemFreeMemoryMb <= lowMemoryThresholdMB) {
+ LOG.warn(
+ "{} is using {} MB > memory limit {} MB and system is low on memory {} free",
+ typeOfCheck,
+ usageMb,
+ memoryLimitMb,
+ systemFreeMemoryMb);
+ return true;
+ }
+
+ //If the system still has some free memory give them a grace period to
+ // drop back down.
+ if (systemFreeMemoryMb < mediumMemoryThresholdMb) {
+ if (memoryLimitExceededStart < 0) {
+ memoryLimitExceededStart = Time.currentTimeMillis();
+ } else {
+ long timeInViolation = Time.currentTimeMillis() - memoryLimitExceededStart;
+ if (timeInViolation > mediumMemoryGracePeriodMs) {
+ LOG.warn(
+ "{} is using {} MB > memory limit {} MB for {} seconds",
+ typeOfCheck,
+ usageMb,
+ memoryLimitMb,
+ timeInViolation / 1000);
+ return true;
+ }
+ }
+ } else {
+ //Otherwise don't bother them
+ LOG.debug("{} is using {} MB > memory limit {} MB", typeOfCheck, usageMb, memoryLimitMb);
+ memoryLimitExceededStart = -1;
+ }
+ } else {
+ memoryLimitExceededStart = -1;
}
- }
- } else {
- //Otherwise don't bother them
- LOG.debug("{} is using {} MB > memory limit {} MB", typeOfCheck, usageMb, memoryLimitMb);
- memoryLimitExceededStart = -1;
}
- } else {
- memoryLimitExceededStart = -1;
- }
+ return false;
}
- return false;
- }
@Override
public long getMemoryUsageMb() {
@@ -883,35 +779,35 @@ public class BasicContainer extends Container {
}
}
- @Override
- public long getMemoryReservationMb() {
- return memoryLimitMB;
- }
+ @Override
+ public long getMemoryReservationMb() {
+ return memoryLimitMB;
+ }
- private long calculateMemoryLimit(final WorkerResources resources, final int memOnHeap) {
- long ret = memOnHeap;
- if (_resourceIsolationManager != null) {
- final int memoffheap = (int) Math.ceil(resources.get_mem_off_heap());
- final int extraMem =
- (int)
- (Math.ceil(
- ObjectReader.getDouble(
- _conf.get(DaemonConfig.STORM_SUPERVISOR_MEMORY_LIMIT_TOLERANCE_MARGIN_MB),
- 0.0)));
- ret += memoffheap + extraMem;
+ private long calculateMemoryLimit(final WorkerResources resources, final int memOnHeap) {
+ long ret = memOnHeap;
+ if (_resourceIsolationManager != null) {
+ final int memoffheap = (int) Math.ceil(resources.get_mem_off_heap());
+ final int extraMem =
+ (int)
+ (Math.ceil(
+ ObjectReader.getDouble(
+ _conf.get(DaemonConfig.STORM_SUPERVISOR_MEMORY_LIMIT_TOLERANCE_MARGIN_MB),
+ 0.0)));
+ ret += memoffheap + extraMem;
+ }
+ return ret;
}
- return ret;
- }
-
+
@Override
public void launch() throws IOException {
_type.assertFull();
LOG.info("Launching worker with assignment {} for this supervisor {} on port {} with id {}", _assignment,
- _supervisorId, _port, _workerId);
+ _supervisorId, _port, _workerId);
String logPrefix = "Worker Process " + _workerId;
ProcessExitCallback processExitCallback = new ProcessExitCallback(logPrefix);
_exitedEarly = false;
-
+
final WorkerResources resources = _assignment.get_resources();
final int memOnHeap = getMemOnHeap(resources);
memoryLimitMB = calculateMemoryLimit(resources, memOnHeap);
@@ -929,7 +825,7 @@ public class BasicContainer extends Container {
if (ld_library_path != null) {
jlp = jlp + System.getProperty("path.separator") + ld_library_path;
}
-
+
topEnvironment.put("LD_LIBRARY_PATH", jlp);
if (_resourceIsolationManager != null) {
@@ -939,11 +835,112 @@ public class BasicContainer extends Container {
}
List<String> commandList = mkLaunchCommand(memOnHeap, stormRoot, jlp);
-
+
LOG.info("Launching worker with command: {}. ", ServerUtils.shellCmd(commandList));
String workerDir = ConfigUtils.workerRoot(_conf, _workerId);
launchWorkerProcess(commandList, topEnvironment, logPrefix, processExitCallback, new File(workerDir));
}
+
+ private static class TopologyMetaData {
+ private final Map<String, Object> _conf;
+ private final String _topologyId;
+ private final AdvancedFSOps _ops;
+ private final String _stormRoot;
+ private boolean _dataCached = false;
+ private List<String> _depLocs = null;
+ private String _stormVersion = null;
+
+ public TopologyMetaData(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, final String stormRoot) {
+ _conf = conf;
+ _topologyId = topologyId;
+ _ops = ops;
+ _stormRoot = stormRoot;
+ }
+
+ public String toString() {
+ List<String> data;
+ String stormVersion;
+ synchronized (this) {
+ data = _depLocs;
+ stormVersion = _stormVersion;
+ }
+ return "META for " + _topologyId + " DEP_LOCS => " + data + " STORM_VERSION => " + stormVersion;
+ }
+
+ private synchronized void readData() throws IOException {
+ final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _ops);
+ final List<String> dependencyLocations = new ArrayList<>();
+ if (stormTopology.get_dependency_jars() != null) {
+ for (String dependency : stormTopology.get_dependency_jars()) {
+ dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath());
+ }
+ }
+
+ if (stormTopology.get_dependency_artifacts() != null) {
+ for (String dependency : stormTopology.get_dependency_artifacts()) {
+ dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath());
+ }
+ }
+ _depLocs = dependencyLocations;
+ _stormVersion = stormTopology.get_storm_version();
+ _dataCached = true;
+ }
+
+ public synchronized List<String> getDepLocs() throws IOException {
+ if (!_dataCached) {
+ readData();
+ }
+ return _depLocs;
+ }
+
+ public synchronized String getStormVersion() throws IOException {
+ if (!_dataCached) {
+ readData();
+ }
+ return _stormVersion;
+ }
+ }
+
+ static class TopoMetaLRUCache {
+ public final int _maxSize = 100; //We could make this configurable in the future...
+
+ @SuppressWarnings("serial")
+ private LinkedHashMap<String, TopologyMetaData> _cache = new LinkedHashMap<String, TopologyMetaData>() {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<String, TopologyMetaData> eldest) {
+ return (size() > _maxSize);
+ }
+ };
+
+ public synchronized TopologyMetaData get(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops,
+ String stormRoot) {
+ //Only go off of the topology id for now.
+ TopologyMetaData dl = _cache.get(topologyId);
+ if (dl == null) {
+ _cache.putIfAbsent(topologyId, new TopologyMetaData(conf, topologyId, ops, stormRoot));
+ dl = _cache.get(topologyId);
+ }
+ return dl;
+ }
+
+ public synchronized void clear() {
+ _cache.clear();
+ }
+ }
+
+ private class ProcessExitCallback implements ExitCodeCallback {
+ private final String _logPrefix;
+
+ public ProcessExitCallback(String logPrefix) {
+ _logPrefix = logPrefix;
+ }
+
+ @Override
+ public void call(int exitCode) {
+ LOG.info("{} exited with code: {}", _logPrefix, exitCode);
+ _exitedEarly = true;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java
index e2043e8..715fd61 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.supervisor;
import java.io.IOException;
import java.util.Map;
-
import org.apache.storm.container.ResourceIsolationInterface;
import org.apache.storm.daemon.supervisor.Container.ContainerType;
import org.apache.storm.generated.LocalAssignment;
@@ -29,13 +23,13 @@ import org.apache.storm.utils.LocalState;
* Launch containers with no security using standard java commands
*/
public class BasicContainerLauncher extends ContainerLauncher {
+ protected final ResourceIsolationInterface _resourceIsolationManager;
private final Map<String, Object> _conf;
private final String _supervisorId;
private final int _supervisorPort;
- protected final ResourceIsolationInterface _resourceIsolationManager;
-
+
public BasicContainerLauncher(Map<String, Object> conf, String supervisorId, int supervisorPort,
- ResourceIsolationInterface resourceIsolationManager) throws IOException {
+ ResourceIsolationInterface resourceIsolationManager) throws IOException {
_conf = conf;
_supervisorId = supervisorId;
_supervisorPort = supervisorPort;
@@ -45,7 +39,7 @@ public class BasicContainerLauncher extends ContainerLauncher {
@Override
public Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
Container container = new BasicContainer(ContainerType.LAUNCH, _conf, _supervisorId, _supervisorPort, port,
- assignment, _resourceIsolationManager, state, null);
+ assignment, _resourceIsolationManager, state, null);
container.setup();
container.launch();
return container;
@@ -54,12 +48,12 @@ public class BasicContainerLauncher extends ContainerLauncher {
@Override
public Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
return new BasicContainer(ContainerType.RECOVER_FULL, _conf, _supervisorId, _supervisorPort, port, assignment,
- _resourceIsolationManager, state, null);
+ _resourceIsolationManager, state, null);
}
@Override
public Killable recoverContainer(String workerId, LocalState localState) throws IOException {
return new BasicContainer(ContainerType.RECOVER_PARTIAL, _conf, _supervisorId, _supervisorPort, -1, null,
- _resourceIsolationManager, localState, workerId);
+ _resourceIsolationManager, localState, workerId);
}
}
[09/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
index c401474..d65c125 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/BlacklistScheduler.java
@@ -1,33 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.blacklist;
import com.google.common.collect.EvictingQueue;
import com.google.common.collect.Sets;
-
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
-
import org.apache.storm.DaemonConfig;
import org.apache.storm.metric.StormMetricsRegistry;
import org.apache.storm.scheduler.Cluster;
@@ -46,29 +38,23 @@ import org.slf4j.LoggerFactory;
public class BlacklistScheduler implements IScheduler {
- private static final Logger LOG = LoggerFactory.getLogger(BlacklistScheduler.class);
-
public static final int DEFAULT_BLACKLIST_SCHEDULER_RESUME_TIME = 1800;
public static final int DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_COUNT = 3;
public static final int DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_TIME = 300;
-
+ private static final Logger LOG = LoggerFactory.getLogger(BlacklistScheduler.class);
private final IScheduler underlyingScheduler;
- private Map<String, Object> conf;
-
protected int toleranceTime;
protected int toleranceCount;
protected int resumeTime;
protected IReporter reporter;
protected IBlacklistStrategy blacklistStrategy;
-
protected int nimbusMonitorFreqSecs;
-
protected Map<String, Set<Integer>> cachedSupervisors;
-
//key is supervisor key ,value is supervisor ports
protected EvictingQueue<HashMap<String, Set<Integer>>> badSupervisorsToleranceSlidingWindow;
protected int windowSize;
protected Set<String> blacklistHost;
+ private Map<String, Object> conf;
public BlacklistScheduler(IScheduler underlyingScheduler) {
this.underlyingScheduler = underlyingScheduler;
@@ -81,18 +67,18 @@ public class BlacklistScheduler implements IScheduler {
this.conf = conf;
toleranceTime = ObjectReader.getInt(this.conf.get(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME),
- DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_TIME);
+ DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_TIME);
toleranceCount = ObjectReader.getInt(this.conf.get(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT),
- DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+ DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
resumeTime = ObjectReader.getInt(this.conf.get(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME),
- DEFAULT_BLACKLIST_SCHEDULER_RESUME_TIME);
+ DEFAULT_BLACKLIST_SCHEDULER_RESUME_TIME);
String reporterClassName = ObjectReader.getString(this.conf.get(DaemonConfig.BLACKLIST_SCHEDULER_REPORTER),
- LogReporter.class.getName());
+ LogReporter.class.getName());
reporter = (IReporter) initializeInstance(reporterClassName, "blacklist reporter");
String strategyClassName = ObjectReader.getString(this.conf.get(DaemonConfig.BLACKLIST_SCHEDULER_STRATEGY),
- DefaultBlacklistStrategy.class.getName());
+ DefaultBlacklistStrategy.class.getName());
blacklistStrategy = (IBlacklistStrategy) initializeInstance(strategyClassName, "blacklist strategy");
nimbusMonitorFreqSecs = ObjectReader.getInt(this.conf.get(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS));
@@ -177,7 +163,7 @@ public class BlacklistScheduler implements IScheduler {
private Set<String> getBlacklistHosts(Cluster cluster, Topologies topologies) {
Set<String> blacklistSet = blacklistStrategy.getBlacklist(new ArrayList<>(badSupervisorsToleranceSlidingWindow),
- cluster, topologies);
+ cluster, topologies);
Set<String> blacklistHostSet = new HashSet<>();
for (String supervisor : blacklistSet) {
String host = cluster.getHost(supervisor);
@@ -235,7 +221,7 @@ public class BlacklistScheduler implements IScheduler {
cachedSupervisors.put(supervisorKey, slots);
}
LOG.info("Worker slot {} was never back to normal during tolerance period, probably dead. Will be removed from cache.",
- workerSlot);
+ workerSlot);
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/IReporter.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/IReporter.java b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/IReporter.java
index 153829c..b1bb933 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/IReporter.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/IReporter.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.blacklist.reporters;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/LogReporter.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/LogReporter.java b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/LogReporter.java
index 3255c9d..4d3eded 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/LogReporter.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/reporters/LogReporter.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.blacklist.reporters;
@@ -21,7 +15,6 @@ package org.apache.storm.scheduler.blacklist.reporters;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,6 +29,6 @@ public class LogReporter implements IReporter {
@Override
public void reportBlacklist(String supervisor, List<Map<String, Set<Integer>>> toleranceBuffer) {
LOG.warn("add supervisor {} to blacklist. The bad slot history of supervisors is : {}",
- supervisor, toleranceBuffer);
+ supervisor, toleranceBuffer);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
index f6ca6ad..3332b2d 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/DefaultBlacklistStrategy.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.blacklist.strategies;
@@ -24,7 +18,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-
import org.apache.storm.DaemonConfig;
import org.apache.storm.scheduler.Cluster;
import org.apache.storm.scheduler.SupervisorDetails;
@@ -39,11 +32,9 @@ import org.slf4j.LoggerFactory;
public class DefaultBlacklistStrategy implements IBlacklistStrategy {
- private static Logger LOG = LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
-
public static final int DEFAULT_BLACKLIST_SCHEDULER_RESUME_TIME = 1800;
public static final int DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_COUNT = 3;
-
+ private static Logger LOG = LoggerFactory.getLogger(DefaultBlacklistStrategy.class);
private IReporter reporter;
private int toleranceCount;
@@ -55,11 +46,11 @@ public class DefaultBlacklistStrategy implements IBlacklistStrategy {
@Override
public void prepare(Map<String, Object> conf) {
toleranceCount = ObjectReader.getInt(conf.get(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT),
- DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
+ DEFAULT_BLACKLIST_SCHEDULER_TOLERANCE_COUNT);
resumeTime = ObjectReader.getInt(conf.get(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME), DEFAULT_BLACKLIST_SCHEDULER_RESUME_TIME);
String reporterClassName = ObjectReader.getString(conf.get(DaemonConfig.BLACKLIST_SCHEDULER_REPORTER),
- LogReporter.class.getName());
+ LogReporter.class.getName());
reporter = (IReporter) initializeInstance(reporterClassName, "blacklist reporter");
nimbusMonitorFreqSecs = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS));
@@ -133,7 +124,7 @@ public class DefaultBlacklistStrategy implements IBlacklistStrategy {
if (shortage > 0) {
LOG.info("total needed num of workers :{}, available num of slots not in blacklist :{}, num blacklist :{}, " +
- "will release some blacklist.", totalNeedNumWorkers, availableSlotsNotInBlacklistCount, blacklist.size());
+ "will release some blacklist.", totalNeedNumWorkers, availableSlotsNotInBlacklistCount, blacklist.size());
//release earliest blacklist
Set<String> readyToRemove = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/IBlacklistStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/IBlacklistStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/IBlacklistStrategy.java
index 8543659..295dd82 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/IBlacklistStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/blacklist/strategies/IBlacklistStrategy.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.blacklist.strategies;
@@ -21,7 +15,6 @@ package org.apache.storm.scheduler.blacklist.strategies;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
import org.apache.storm.scheduler.Cluster;
import org.apache.storm.scheduler.Topologies;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/DefaultPool.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/DefaultPool.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/DefaultPool.java
index b690764..1b6ac91 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/DefaultPool.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/DefaultPool.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.multitenant;
@@ -24,196 +18,199 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.storm.scheduler.SchedulerAssignment;
import org.apache.storm.scheduler.TopologyDetails;
import org.apache.storm.scheduler.WorkerSlot;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A pool of machines that anyone can use, but topologies are not isolated
*/
public class DefaultPool extends NodePool {
- private static final Logger LOG = LoggerFactory.getLogger(DefaultPool.class);
- private Set<Node> _nodes = new HashSet<>();
- private HashMap<String, TopologyDetails> _tds = new HashMap<>();
-
- @Override
- public void addTopology(TopologyDetails td) {
- String topId = td.getId();
- LOG.debug("Adding in Topology {}", topId);
- _tds.put(topId, td);
- SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
- if (assignment != null) {
- for (WorkerSlot ws: assignment.getSlots()) {
- Node n = _nodeIdToNode.get(ws.getNodeId());
- _nodes.add(n);
- }
- }
- }
-
- @Override
- public boolean canAdd(TopologyDetails td) {
- return true;
- }
-
- @Override
- public Collection<Node> takeNodes(int nodesNeeded) {
- HashSet<Node> ret = new HashSet<>();
- LinkedList<Node> sortedNodes = new LinkedList<>(_nodes);
- Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC);
- for (Node n: sortedNodes) {
- if (nodesNeeded <= ret.size()) {
- break;
- }
- if (n.isAlive()) {
- n.freeAllSlots(_cluster);
- _nodes.remove(n);
- ret.add(n);
- }
- }
- return ret;
- }
-
- @Override
- public int nodesAvailable() {
- int total = 0;
- for (Node n: _nodes) {
- if (n.isAlive()) total++;
- }
- return total;
- }
-
- @Override
- public int slotsAvailable() {
- return Node.countTotalSlotsAlive(_nodes);
- }
-
- @Override
- public NodeAndSlotCounts getNodeAndSlotCountIfSlotsWereTaken(int slotsNeeded) {
- int nodesFound = 0;
- int slotsFound = 0;
- LinkedList<Node> sortedNodes = new LinkedList<>(_nodes);
- Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC);
- for (Node n: sortedNodes) {
- if (slotsNeeded <= 0) {
- break;
- }
- if (n.isAlive()) {
- nodesFound++;
- int totalSlotsFree = n.totalSlots();
- slotsFound += totalSlotsFree;
- slotsNeeded -= totalSlotsFree;
- }
+ private static final Logger LOG = LoggerFactory.getLogger(DefaultPool.class);
+ private Set<Node> _nodes = new HashSet<>();
+ private HashMap<String, TopologyDetails> _tds = new HashMap<>();
+
+ @Override
+ public void addTopology(TopologyDetails td) {
+ String topId = td.getId();
+ LOG.debug("Adding in Topology {}", topId);
+ _tds.put(topId, td);
+ SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
+ if (assignment != null) {
+ for (WorkerSlot ws : assignment.getSlots()) {
+ Node n = _nodeIdToNode.get(ws.getNodeId());
+ _nodes.add(n);
+ }
+ }
}
- return new NodeAndSlotCounts(nodesFound, slotsFound);
- }
-
- @Override
- public Collection<Node> takeNodesBySlots(int slotsNeeded) {
- HashSet<Node> ret = new HashSet<>();
- LinkedList<Node> sortedNodes = new LinkedList<>(_nodes);
- Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC);
- for (Node n: sortedNodes) {
- if (slotsNeeded <= 0) {
- break;
- }
- if (n.isAlive()) {
- n.freeAllSlots(_cluster);
- _nodes.remove(n);
- ret.add(n);
- slotsNeeded -= n.totalSlotsFree();
- }
+
+ @Override
+ public boolean canAdd(TopologyDetails td) {
+ return true;
}
- return ret;
- }
-
- @Override
- public void scheduleAsNeeded(NodePool... lesserPools) {
- for (TopologyDetails td : _tds.values()) {
- String topId = td.getId();
- if (_cluster.needsScheduling(td)) {
- LOG.debug("Scheduling topology {}",topId);
- int totalTasks = td.getExecutors().size();
- int origRequest = td.getNumWorkers();
- int slotsRequested = Math.min(totalTasks, origRequest);
- int slotsUsed = Node.countSlotsUsed(topId, _nodes);
- int slotsFree = Node.countFreeSlotsAlive(_nodes);
- //Check to see if we have enough slots before trying to get them
- int slotsAvailable = 0;
- if (slotsRequested > slotsFree) {
- slotsAvailable = NodePool.slotsAvailable(lesserPools);
- }
- int slotsToUse = Math.min(slotsRequested - slotsUsed, slotsFree + slotsAvailable);
- int executorsNotRunning = _cluster.getUnassignedExecutors(td).size();
- LOG.debug("Slots... requested {} used {} free {} available {} to be used {}, executors not running {}",
- slotsRequested, slotsUsed, slotsFree, slotsAvailable, slotsToUse, executorsNotRunning);
- if (slotsToUse <= 0) {
- if (executorsNotRunning > 0) {
- _cluster.setStatus(topId,"Not fully scheduled (No free slots in default pool) "+executorsNotRunning+" executors not scheduled");
- } else {
- if (slotsUsed < slotsRequested) {
- _cluster.setStatus(topId,"Running with fewer slots than requested ("+slotsUsed+"/"+origRequest+")");
- } else { //slotsUsed < origRequest
- _cluster.setStatus(topId,"Fully Scheduled (requested "+origRequest+" slots, but could only use "+slotsUsed+")");
+
+ @Override
+ public Collection<Node> takeNodes(int nodesNeeded) {
+ HashSet<Node> ret = new HashSet<>();
+ LinkedList<Node> sortedNodes = new LinkedList<>(_nodes);
+ Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC);
+ for (Node n : sortedNodes) {
+ if (nodesNeeded <= ret.size()) {
+ break;
+ }
+ if (n.isAlive()) {
+ n.freeAllSlots(_cluster);
+ _nodes.remove(n);
+ ret.add(n);
}
- }
- continue;
}
+ return ret;
+ }
- int slotsNeeded = slotsToUse - slotsFree;
- if (slotsNeeded > 0) {
- _nodes.addAll(NodePool.takeNodesBySlot(slotsNeeded, lesserPools));
+ @Override
+ public int nodesAvailable() {
+ int total = 0;
+ for (Node n : _nodes) {
+ if (n.isAlive()) total++;
}
+ return total;
+ }
+
+ @Override
+ public int slotsAvailable() {
+ return Node.countTotalSlotsAlive(_nodes);
+ }
- if (executorsNotRunning <= 0) {
- //There are free slots that we can take advantage of now.
- for (Node n: _nodes) {
- n.freeTopology(topId, _cluster);
- }
- slotsFree = Node.countFreeSlotsAlive(_nodes);
- slotsToUse = Math.min(slotsRequested, slotsFree);
+ @Override
+ public NodeAndSlotCounts getNodeAndSlotCountIfSlotsWereTaken(int slotsNeeded) {
+ int nodesFound = 0;
+ int slotsFound = 0;
+ LinkedList<Node> sortedNodes = new LinkedList<>(_nodes);
+ Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC);
+ for (Node n : sortedNodes) {
+ if (slotsNeeded <= 0) {
+ break;
+ }
+ if (n.isAlive()) {
+ nodesFound++;
+ int totalSlotsFree = n.totalSlots();
+ slotsFound += totalSlotsFree;
+ slotsNeeded -= totalSlotsFree;
+ }
}
-
- RoundRobinSlotScheduler slotSched =
- new RoundRobinSlotScheduler(td, slotsToUse, _cluster);
-
- LinkedList<Node> nodes = new LinkedList<>(_nodes);
- while (true) {
- Node n;
- do {
- if (nodes.isEmpty()) {
- throw new IllegalStateException("This should not happen, we" +
- " messed up and did not get enough slots");
+ return new NodeAndSlotCounts(nodesFound, slotsFound);
+ }
+
+ @Override
+ public Collection<Node> takeNodesBySlots(int slotsNeeded) {
+ HashSet<Node> ret = new HashSet<>();
+ LinkedList<Node> sortedNodes = new LinkedList<>(_nodes);
+ Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC);
+ for (Node n : sortedNodes) {
+ if (slotsNeeded <= 0) {
+ break;
}
- n = nodes.peekFirst();
- if (n.totalSlotsFree() == 0) {
- nodes.remove();
- n = null;
+ if (n.isAlive()) {
+ n.freeAllSlots(_cluster);
+ _nodes.remove(n);
+ ret.add(n);
+ slotsNeeded -= n.totalSlotsFree();
}
- } while (n == null);
- if (!slotSched.assignSlotTo(n)) {
- break;
- }
}
- int afterSchedSlotsUsed = Node.countSlotsUsed(topId, _nodes);
- if (afterSchedSlotsUsed < slotsRequested) {
- _cluster.setStatus(topId,"Running with fewer slots than requested ("+afterSchedSlotsUsed+"/"+origRequest+")");
- } else if (afterSchedSlotsUsed < origRequest) {
- _cluster.setStatus(topId,"Fully Scheduled (requested "+origRequest+" slots, but could only use "+afterSchedSlotsUsed+")");
- } else {
- _cluster.setStatus(topId,"Fully Scheduled");
+ return ret;
+ }
+
+ @Override
+ public void scheduleAsNeeded(NodePool... lesserPools) {
+ for (TopologyDetails td : _tds.values()) {
+ String topId = td.getId();
+ if (_cluster.needsScheduling(td)) {
+ LOG.debug("Scheduling topology {}", topId);
+ int totalTasks = td.getExecutors().size();
+ int origRequest = td.getNumWorkers();
+ int slotsRequested = Math.min(totalTasks, origRequest);
+ int slotsUsed = Node.countSlotsUsed(topId, _nodes);
+ int slotsFree = Node.countFreeSlotsAlive(_nodes);
+ //Check to see if we have enough slots before trying to get them
+ int slotsAvailable = 0;
+ if (slotsRequested > slotsFree) {
+ slotsAvailable = NodePool.slotsAvailable(lesserPools);
+ }
+ int slotsToUse = Math.min(slotsRequested - slotsUsed, slotsFree + slotsAvailable);
+ int executorsNotRunning = _cluster.getUnassignedExecutors(td).size();
+ LOG.debug("Slots... requested {} used {} free {} available {} to be used {}, executors not running {}",
+ slotsRequested, slotsUsed, slotsFree, slotsAvailable, slotsToUse, executorsNotRunning);
+ if (slotsToUse <= 0) {
+ if (executorsNotRunning > 0) {
+ _cluster.setStatus(topId, "Not fully scheduled (No free slots in default pool) " + executorsNotRunning +
+ " executors not scheduled");
+ } else {
+ if (slotsUsed < slotsRequested) {
+ _cluster.setStatus(topId, "Running with fewer slots than requested (" + slotsUsed + "/" + origRequest + ")");
+ } else { //slotsUsed < origRequest
+ _cluster.setStatus(topId,
+ "Fully Scheduled (requested " + origRequest + " slots, but could only use " + slotsUsed +
+ ")");
+ }
+ }
+ continue;
+ }
+
+ int slotsNeeded = slotsToUse - slotsFree;
+ if (slotsNeeded > 0) {
+ _nodes.addAll(NodePool.takeNodesBySlot(slotsNeeded, lesserPools));
+ }
+
+ if (executorsNotRunning <= 0) {
+ //There are free slots that we can take advantage of now.
+ for (Node n : _nodes) {
+ n.freeTopology(topId, _cluster);
+ }
+ slotsFree = Node.countFreeSlotsAlive(_nodes);
+ slotsToUse = Math.min(slotsRequested, slotsFree);
+ }
+
+ RoundRobinSlotScheduler slotSched =
+ new RoundRobinSlotScheduler(td, slotsToUse, _cluster);
+
+ LinkedList<Node> nodes = new LinkedList<>(_nodes);
+ while (true) {
+ Node n;
+ do {
+ if (nodes.isEmpty()) {
+ throw new IllegalStateException("This should not happen, we" +
+ " messed up and did not get enough slots");
+ }
+ n = nodes.peekFirst();
+ if (n.totalSlotsFree() == 0) {
+ nodes.remove();
+ n = null;
+ }
+ } while (n == null);
+ if (!slotSched.assignSlotTo(n)) {
+ break;
+ }
+ }
+ int afterSchedSlotsUsed = Node.countSlotsUsed(topId, _nodes);
+ if (afterSchedSlotsUsed < slotsRequested) {
+ _cluster.setStatus(topId, "Running with fewer slots than requested (" + afterSchedSlotsUsed + "/" + origRequest + ")");
+ } else if (afterSchedSlotsUsed < origRequest) {
+ _cluster.setStatus(topId,
+ "Fully Scheduled (requested " + origRequest + " slots, but could only use " + afterSchedSlotsUsed +
+ ")");
+ } else {
+ _cluster.setStatus(topId, "Fully Scheduled");
+ }
+ } else {
+ _cluster.setStatus(topId, "Fully Scheduled");
+ }
}
- } else {
- _cluster.setStatus(topId,"Fully Scheduled");
- }
}
- }
-
- @Override
- public String toString() {
- return "DefaultPool " + _nodes.size() + " nodes " + _tds.size() + " topologies";
- }
+
+ @Override
+ public String toString() {
+ return "DefaultPool " + _nodes.size() + " nodes " + _tds.size() + " topologies";
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/FreePool.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/FreePool.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/FreePool.java
index 88cec6b..e868223 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/FreePool.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/FreePool.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.multitenant;
@@ -23,103 +17,101 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.storm.scheduler.Cluster;
import org.apache.storm.scheduler.TopologyDetails;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* All of the machines that currently have nothing assigned to them
*/
public class FreePool extends NodePool {
- private static final Logger LOG = LoggerFactory.getLogger(FreePool.class);
- private Set<Node> _nodes = new HashSet<>();
- private int _totalSlots = 0;
+ private static final Logger LOG = LoggerFactory.getLogger(FreePool.class);
+ private Set<Node> _nodes = new HashSet<>();
+ private int _totalSlots = 0;
- @Override
- public void init(Cluster cluster, Map<String, Node> nodeIdToNode) {
- super.init(cluster, nodeIdToNode);
- for (Node n: nodeIdToNode.values()) {
- if(n.isTotallyFree() && n.isAlive()) {
- _nodes.add(n);
- _totalSlots += n.totalSlotsFree();
- }
+ @Override
+ public void init(Cluster cluster, Map<String, Node> nodeIdToNode) {
+ super.init(cluster, nodeIdToNode);
+ for (Node n : nodeIdToNode.values()) {
+ if (n.isTotallyFree() && n.isAlive()) {
+ _nodes.add(n);
+ _totalSlots += n.totalSlotsFree();
+ }
+ }
+ LOG.debug("Found {} nodes with {} slots", _nodes.size(), _totalSlots);
}
- LOG.debug("Found {} nodes with {} slots", _nodes.size(), _totalSlots);
- }
-
- @Override
- public void addTopology(TopologyDetails td) {
- throw new IllegalArgumentException("The free pool cannot run any topologies");
- }
- @Override
- public boolean canAdd(TopologyDetails td) {
- // The free pool never has anything running
- return false;
- }
-
- @Override
- public Collection<Node> takeNodes(int nodesNeeded) {
- HashSet<Node> ret = new HashSet<>();
- Iterator<Node> it = _nodes.iterator();
- while (it.hasNext() && nodesNeeded > ret.size()) {
- Node n = it.next();
- ret.add(n);
- _totalSlots -= n.totalSlotsFree();
- it.remove();
+ @Override
+ public void addTopology(TopologyDetails td) {
+ throw new IllegalArgumentException("The free pool cannot run any topologies");
}
- return ret;
- }
-
- @Override
- public int nodesAvailable() {
- return _nodes.size();
- }
- @Override
- public int slotsAvailable() {
- return _totalSlots;
- }
+ @Override
+ public boolean canAdd(TopologyDetails td) {
+ // The free pool never has anything running
+ return false;
+ }
- @Override
- public Collection<Node> takeNodesBySlots(int slotsNeeded) {
- HashSet<Node> ret = new HashSet<>();
- Iterator<Node> it = _nodes.iterator();
- while (it.hasNext() && slotsNeeded > 0) {
- Node n = it.next();
- ret.add(n);
- _totalSlots -= n.totalSlotsFree();
- slotsNeeded -= n.totalSlotsFree();
- it.remove();
+ @Override
+ public Collection<Node> takeNodes(int nodesNeeded) {
+ HashSet<Node> ret = new HashSet<>();
+ Iterator<Node> it = _nodes.iterator();
+ while (it.hasNext() && nodesNeeded > ret.size()) {
+ Node n = it.next();
+ ret.add(n);
+ _totalSlots -= n.totalSlotsFree();
+ it.remove();
+ }
+ return ret;
}
- return ret;
- }
-
- @Override
- public NodeAndSlotCounts getNodeAndSlotCountIfSlotsWereTaken(int slotsNeeded) {
- int slotsFound = 0;
- int nodesFound = 0;
- Iterator<Node> it = _nodes.iterator();
- while (it.hasNext() && slotsNeeded > 0) {
- Node n = it.next();
- nodesFound++;
- int totalSlots = n.totalSlots();
- slotsFound += totalSlots;
- slotsNeeded -= totalSlots;
+
+ @Override
+ public int nodesAvailable() {
+ return _nodes.size();
}
- return new NodeAndSlotCounts(nodesFound, slotsFound);
- }
- @Override
- public void scheduleAsNeeded(NodePool... lesserPools) {
- //No topologies running so NOOP
- }
-
- @Override
- public String toString() {
- return "FreePool of "+_nodes.size()+" nodes with "+_totalSlots+" slots";
- }
+ @Override
+ public int slotsAvailable() {
+ return _totalSlots;
+ }
+
+ @Override
+ public Collection<Node> takeNodesBySlots(int slotsNeeded) {
+ HashSet<Node> ret = new HashSet<>();
+ Iterator<Node> it = _nodes.iterator();
+ while (it.hasNext() && slotsNeeded > 0) {
+ Node n = it.next();
+ ret.add(n);
+ _totalSlots -= n.totalSlotsFree();
+ slotsNeeded -= n.totalSlotsFree();
+ it.remove();
+ }
+ return ret;
+ }
+
+ @Override
+ public NodeAndSlotCounts getNodeAndSlotCountIfSlotsWereTaken(int slotsNeeded) {
+ int slotsFound = 0;
+ int nodesFound = 0;
+ Iterator<Node> it = _nodes.iterator();
+ while (it.hasNext() && slotsNeeded > 0) {
+ Node n = it.next();
+ nodesFound++;
+ int totalSlots = n.totalSlots();
+ slotsFound += totalSlots;
+ slotsNeeded -= totalSlots;
+ }
+ return new NodeAndSlotCounts(nodesFound, slotsFound);
+ }
+
+ @Override
+ public void scheduleAsNeeded(NodePool... lesserPools) {
+ //No topologies running so NOOP
+ }
+
+ @Override
+ public String toString() {
+ return "FreePool of " + _nodes.size() + " nodes with " + _totalSlots + " slots";
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/IsolatedPool.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/IsolatedPool.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/IsolatedPool.java
index f16e4fa..19ce712 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/IsolatedPool.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/IsolatedPool.java
@@ -1,371 +1,362 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.multitenant;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.storm.Config;
import org.apache.storm.scheduler.SchedulerAssignment;
import org.apache.storm.scheduler.TopologyDetails;
import org.apache.storm.scheduler.WorkerSlot;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A pool of machines that can be used to run isolated topologies
*/
public class IsolatedPool extends NodePool {
- private static final Logger LOG = LoggerFactory.getLogger(IsolatedPool.class);
- private Map<String, Set<Node>> _topologyIdToNodes = new HashMap<>();
- private HashMap<String, TopologyDetails> _tds = new HashMap<>();
- private HashSet<String> _isolated = new HashSet<>();
- private int _maxNodes;
- private int _usedNodes;
+ private static final Logger LOG = LoggerFactory.getLogger(IsolatedPool.class);
+ private Map<String, Set<Node>> _topologyIdToNodes = new HashMap<>();
+ private HashMap<String, TopologyDetails> _tds = new HashMap<>();
+ private HashSet<String> _isolated = new HashSet<>();
+ private int _maxNodes;
+ private int _usedNodes;
- public IsolatedPool(int maxNodes) {
- _maxNodes = maxNodes;
- _usedNodes = 0;
- }
-
- @Override
- public void addTopology(TopologyDetails td) {
- String topId = td.getId();
- LOG.debug("Adding in Topology {}", topId);
- SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
- Set<Node> assignedNodes = new HashSet<>();
- if (assignment != null) {
- for (WorkerSlot ws: assignment.getSlots()) {
- Node n = _nodeIdToNode.get(ws.getNodeId());
- assignedNodes.add(n);
- }
- }
- _usedNodes += assignedNodes.size();
- _topologyIdToNodes.put(topId, assignedNodes);
- _tds.put(topId, td);
- if (td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES) != null) {
- _isolated.add(topId);
+ public IsolatedPool(int maxNodes) {
+ _maxNodes = maxNodes;
+ _usedNodes = 0;
}
- }
- @Override
- public boolean canAdd(TopologyDetails td) {
- //Only add topologies that are not sharing nodes with other topologies
- String topId = td.getId();
- SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
- if (assignment != null) {
- for (WorkerSlot ws: assignment.getSlots()) {
- Node n = _nodeIdToNode.get(ws.getNodeId());
- if (n.getRunningTopologies().size() > 1) {
- return false;
- }
- }
- }
- return true;
- }
-
- @Override
- public void scheduleAsNeeded(NodePool ... lesserPools) {
- for (String topId : _topologyIdToNodes.keySet()) {
- TopologyDetails td = _tds.get(topId);
- Set<Node> allNodes = _topologyIdToNodes.get(topId);
- Number nodesRequested = (Number) td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES);
- Integer effectiveNodesRequested = null;
- if (nodesRequested != null) {
- effectiveNodesRequested = Math.min(td.getExecutors().size(),
- +nodesRequested.intValue());
- }
- if (_cluster.needsScheduling(td) ||
- (effectiveNodesRequested != null &&
- allNodes.size() != effectiveNodesRequested)) {
- LOG.debug("Scheduling topology {}", topId);
- int slotsToUse = 0;
- if (effectiveNodesRequested == null) {
- slotsToUse = getNodesForNotIsolatedTop(td, allNodes, lesserPools);
- } else {
- slotsToUse = getNodesForIsolatedTop(td, allNodes, lesserPools,
- effectiveNodesRequested);
+ @Override
+ public void addTopology(TopologyDetails td) {
+ String topId = td.getId();
+ LOG.debug("Adding in Topology {}", topId);
+ SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
+ Set<Node> assignedNodes = new HashSet<>();
+ if (assignment != null) {
+ for (WorkerSlot ws : assignment.getSlots()) {
+ Node n = _nodeIdToNode.get(ws.getNodeId());
+ assignedNodes.add(n);
+ }
}
- //No slots to schedule for some reason, so skip it.
- if (slotsToUse <= 0) {
- continue;
+ _usedNodes += assignedNodes.size();
+ _topologyIdToNodes.put(topId, assignedNodes);
+ _tds.put(topId, td);
+ if (td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES) != null) {
+ _isolated.add(topId);
}
-
- RoundRobinSlotScheduler slotSched =
- new RoundRobinSlotScheduler(td, slotsToUse, _cluster);
+ }
- LOG.debug("Nodes sorted by free space {}", allNodes);
- while (true) {
- Node n = findBestNode(allNodes);
- if (n == null) {
- LOG.error("No nodes to use to assign topology {}", td.getName());
- break;
- }
- if (!slotSched.assignSlotTo(n)) {
- break;
- }
+ @Override
+ public boolean canAdd(TopologyDetails td) {
+ //Only add topologies that are not sharing nodes with other topologies
+ String topId = td.getId();
+ SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
+ if (assignment != null) {
+ for (WorkerSlot ws : assignment.getSlots()) {
+ Node n = _nodeIdToNode.get(ws.getNodeId());
+ if (n.getRunningTopologies().size() > 1) {
+ return false;
+ }
+ }
}
- }
- Set<Node> found = _topologyIdToNodes.get(topId);
- int nc = found == null ? 0 : found.size();
- _cluster.setStatus(topId,"Scheduled Isolated on "+nc+" Nodes");
+ return true;
}
- }
- private Node findBestNode(Collection<Node> nodes) {
- Node ret = null;
- for(Node node : nodes) {
- if(ret == null ) {
- if(node.totalSlotsFree() > 0) {
- ret = node;
- }
- } else {
- if (node.totalSlotsFree() > 0) {
- if (node.totalSlotsUsed() < ret.totalSlotsUsed()) {
- ret = node;
- } else if (node.totalSlotsUsed() == ret.totalSlotsUsed()) {
- if(node.totalSlotsFree() > ret.totalSlotsFree()) {
- ret = node;
+ @Override
+ public void scheduleAsNeeded(NodePool... lesserPools) {
+ for (String topId : _topologyIdToNodes.keySet()) {
+ TopologyDetails td = _tds.get(topId);
+ Set<Node> allNodes = _topologyIdToNodes.get(topId);
+ Number nodesRequested = (Number) td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES);
+ Integer effectiveNodesRequested = null;
+ if (nodesRequested != null) {
+ effectiveNodesRequested = Math.min(td.getExecutors().size(),
+ +nodesRequested.intValue());
+ }
+ if (_cluster.needsScheduling(td) ||
+ (effectiveNodesRequested != null &&
+ allNodes.size() != effectiveNodesRequested)) {
+ LOG.debug("Scheduling topology {}", topId);
+ int slotsToUse = 0;
+ if (effectiveNodesRequested == null) {
+ slotsToUse = getNodesForNotIsolatedTop(td, allNodes, lesserPools);
+ } else {
+ slotsToUse = getNodesForIsolatedTop(td, allNodes, lesserPools,
+ effectiveNodesRequested);
+ }
+ //No slots to schedule for some reason, so skip it.
+ if (slotsToUse <= 0) {
+ continue;
+ }
+
+ RoundRobinSlotScheduler slotSched =
+ new RoundRobinSlotScheduler(td, slotsToUse, _cluster);
+
+ LOG.debug("Nodes sorted by free space {}", allNodes);
+ while (true) {
+ Node n = findBestNode(allNodes);
+ if (n == null) {
+ LOG.error("No nodes to use to assign topology {}", td.getName());
+ break;
+ }
+ if (!slotSched.assignSlotTo(n)) {
+ break;
+ }
+ }
}
- }
+ Set<Node> found = _topologyIdToNodes.get(topId);
+ int nc = found == null ? 0 : found.size();
+ _cluster.setStatus(topId, "Scheduled Isolated on " + nc + " Nodes");
}
- }
}
- return ret;
- }
-
- /**
- * Get the nodes needed to schedule an isolated topology.
- * @param td the topology to be scheduled
- * @param allNodes the nodes already scheduled for this topology.
- * This will be updated to include new nodes if needed.
- * @param lesserPools node pools we can steal nodes from
- * @return the number of additional slots that should be used for scheduling.
- */
- private int getNodesForIsolatedTop(TopologyDetails td, Set<Node> allNodes,
- NodePool[] lesserPools, int nodesRequested) {
- String topId = td.getId();
- LOG.debug("Topology {} is isolated", topId);
- int nodesFromUsAvailable = nodesAvailable();
- int nodesFromOthersAvailable = NodePool.nodesAvailable(lesserPools);
- int nodesUsed = _topologyIdToNodes.get(topId).size();
- int nodesNeeded = nodesRequested - nodesUsed;
- LOG.debug("Nodes... requested {} used {} available from us {} " +
- "avail from other {} needed {}", nodesRequested,
- nodesUsed, nodesFromUsAvailable, nodesFromOthersAvailable,
- nodesNeeded);
- if ((nodesNeeded - nodesFromUsAvailable) > (_maxNodes - _usedNodes)) {
- _cluster.setStatus(topId,"Max Nodes("+_maxNodes+") for this user would be exceeded. "
- + ((nodesNeeded - nodesFromUsAvailable) - (_maxNodes - _usedNodes))
- + " more nodes needed to run topology.");
- return 0;
+ private Node findBestNode(Collection<Node> nodes) {
+ Node ret = null;
+ for (Node node : nodes) {
+ if (ret == null) {
+ if (node.totalSlotsFree() > 0) {
+ ret = node;
+ }
+ } else {
+ if (node.totalSlotsFree() > 0) {
+ if (node.totalSlotsUsed() < ret.totalSlotsUsed()) {
+ ret = node;
+ } else if (node.totalSlotsUsed() == ret.totalSlotsUsed()) {
+ if (node.totalSlotsFree() > ret.totalSlotsFree()) {
+ ret = node;
+ }
+ }
+ }
+ }
+ }
+ return ret;
}
- //In order to avoid going over _maxNodes I may need to steal from
- // myself even though other pools have free nodes. so figure out how
- // much each group should provide
- int nodesNeededFromOthers = Math.min(Math.min(_maxNodes - _usedNodes,
- nodesFromOthersAvailable), nodesNeeded);
- int nodesNeededFromUs = nodesNeeded - nodesNeededFromOthers;
- LOG.debug("Nodes... needed from us {} needed from others {}",
- nodesNeededFromUs, nodesNeededFromOthers);
+ /**
+ * Get the nodes needed to schedule an isolated topology.
+ * @param td the topology to be scheduled
+ * @param allNodes the nodes already scheduled for this topology.
+ * This will be updated to include new nodes if needed.
+ * @param lesserPools node pools we can steal nodes from
+ * @return the number of additional slots that should be used for scheduling.
+ */
+ private int getNodesForIsolatedTop(TopologyDetails td, Set<Node> allNodes,
+ NodePool[] lesserPools, int nodesRequested) {
+ String topId = td.getId();
+ LOG.debug("Topology {} is isolated", topId);
+ int nodesFromUsAvailable = nodesAvailable();
+ int nodesFromOthersAvailable = NodePool.nodesAvailable(lesserPools);
- if (nodesNeededFromUs > nodesFromUsAvailable) {
- _cluster.setStatus(topId, "Not Enough Nodes Available to Schedule Topology");
- return 0;
- }
+ int nodesUsed = _topologyIdToNodes.get(topId).size();
+ int nodesNeeded = nodesRequested - nodesUsed;
+ LOG.debug("Nodes... requested {} used {} available from us {} " +
+ "avail from other {} needed {}", nodesRequested,
+ nodesUsed, nodesFromUsAvailable, nodesFromOthersAvailable,
+ nodesNeeded);
+ if ((nodesNeeded - nodesFromUsAvailable) > (_maxNodes - _usedNodes)) {
+ _cluster.setStatus(topId, "Max Nodes(" + _maxNodes + ") for this user would be exceeded. "
+ + ((nodesNeeded - nodesFromUsAvailable) - (_maxNodes - _usedNodes))
+ + " more nodes needed to run topology.");
+ return 0;
+ }
- //Get the nodes
- Collection<Node> found = NodePool.takeNodes(nodesNeededFromOthers, lesserPools);
- _usedNodes += found.size();
- allNodes.addAll(found);
- Collection<Node> foundMore = takeNodes(nodesNeededFromUs);
- _usedNodes += foundMore.size();
- allNodes.addAll(foundMore);
+ //In order to avoid going over _maxNodes I may need to steal from
+ // myself even though other pools have free nodes. so figure out how
+ // much each group should provide
+ int nodesNeededFromOthers = Math.min(Math.min(_maxNodes - _usedNodes,
+ nodesFromOthersAvailable), nodesNeeded);
+ int nodesNeededFromUs = nodesNeeded - nodesNeededFromOthers;
+ LOG.debug("Nodes... needed from us {} needed from others {}",
+ nodesNeededFromUs, nodesNeededFromOthers);
- int totalTasks = td.getExecutors().size();
- int origRequest = td.getNumWorkers();
- int slotsRequested = Math.min(totalTasks, origRequest);
- int slotsUsed = Node.countSlotsUsed(allNodes);
- int slotsFree = Node.countFreeSlotsAlive(allNodes);
- int slotsToUse = Math.min(slotsRequested - slotsUsed, slotsFree);
- if (slotsToUse <= 0) {
- // if # of workers requested is more than we currently have
- if (origRequest > slotsUsed) {
- _cluster.setStatus(topId, "Running with fewer slots than requested " + slotsUsed + "/" +
- origRequest + " on " + allNodes.size() + " node(s) with " + (slotsUsed + slotsFree) + " total slots");
- } else {
- // if # of workers requested is less than we took
- // then we know some workers we track died, since we have more workers than we are supposed to have
- _cluster.setStatus(topId, "Node has partially crashed, if this situation persists rebalance the topology.");
+ if (nodesNeededFromUs > nodesFromUsAvailable) {
+ _cluster.setStatus(topId, "Not Enough Nodes Available to Schedule Topology");
+ return 0;
}
+
+ //Get the nodes
+ Collection<Node> found = NodePool.takeNodes(nodesNeededFromOthers, lesserPools);
+ _usedNodes += found.size();
+ allNodes.addAll(found);
+ Collection<Node> foundMore = takeNodes(nodesNeededFromUs);
+ _usedNodes += foundMore.size();
+ allNodes.addAll(foundMore);
+
+ int totalTasks = td.getExecutors().size();
+ int origRequest = td.getNumWorkers();
+ int slotsRequested = Math.min(totalTasks, origRequest);
+ int slotsUsed = Node.countSlotsUsed(allNodes);
+ int slotsFree = Node.countFreeSlotsAlive(allNodes);
+ int slotsToUse = Math.min(slotsRequested - slotsUsed, slotsFree);
+ if (slotsToUse <= 0) {
+ // if # of workers requested is more than we currently have
+ if (origRequest > slotsUsed) {
+ _cluster.setStatus(topId, "Running with fewer slots than requested " + slotsUsed + "/" +
+ origRequest + " on " + allNodes.size() + " node(s) with " + (slotsUsed + slotsFree) +
+ " total slots");
+ } else {
+ // if # of workers requested is less than we took
+ // then we know some workers we track died, since we have more workers than we are supposed to have
+ _cluster.setStatus(topId, "Node has partially crashed, if this situation persists rebalance the topology.");
+ }
+ }
+ return slotsToUse;
}
- return slotsToUse;
- }
-
- /**
- * Get the nodes needed to schedule a non-isolated topology.
- * @param td the topology to be scheduled
- * @param allNodes the nodes already scheduled for this topology.
- * This will be updated to include new nodes if needed.
- * @param lesserPools node pools we can steal nodes from
- * @return the number of additional slots that should be used for scheduling.
- */
- private int getNodesForNotIsolatedTop(TopologyDetails td, Set<Node> allNodes,
- NodePool[] lesserPools) {
- String topId = td.getId();
- LOG.debug("Topology {} is not isolated",topId);
- int totalTasks = td.getExecutors().size();
- int origRequest = td.getNumWorkers();
- int slotsRequested = Math.min(totalTasks, origRequest);
- int slotsUsed = Node.countSlotsUsed(topId, allNodes);
- int slotsFree = Node.countFreeSlotsAlive(allNodes);
- //Check to see if we have enough slots before trying to get them
- int slotsAvailable = 0;
- if (slotsRequested > slotsFree) {
- slotsAvailable = NodePool.slotsAvailable(lesserPools);
- }
- int slotsToUse = Math.min(slotsRequested - slotsUsed, slotsFree + slotsAvailable);
- LOG.debug("Slots... requested {} used {} free {} available {} to be used {}",
- slotsRequested, slotsUsed, slotsFree, slotsAvailable, slotsToUse);
- if (slotsToUse <= 0) {
- _cluster.setStatus(topId, "Not Enough Slots Available to Schedule Topology");
- return 0;
- }
- int slotsNeeded = slotsToUse - slotsFree;
- int numNewNodes = NodePool.getNodeCountIfSlotsWereTaken(slotsNeeded, lesserPools);
- LOG.debug("Nodes... new {} used {} max {}",
- numNewNodes, _usedNodes, _maxNodes);
- if ((numNewNodes + _usedNodes) > _maxNodes) {
- _cluster.setStatus(topId,"Max Nodes("+_maxNodes+") for this user would be exceeded. " +
- (numNewNodes - (_maxNodes - _usedNodes)) + " more nodes needed to run topology.");
- return 0;
+
+ /**
+ * Get the nodes needed to schedule a non-isolated topology.
+ * @param td the topology to be scheduled
+ * @param allNodes the nodes already scheduled for this topology.
+ * This will be updated to include new nodes if needed.
+ * @param lesserPools node pools we can steal nodes from
+ * @return the number of additional slots that should be used for scheduling.
+ */
+ private int getNodesForNotIsolatedTop(TopologyDetails td, Set<Node> allNodes,
+ NodePool[] lesserPools) {
+ String topId = td.getId();
+ LOG.debug("Topology {} is not isolated", topId);
+ int totalTasks = td.getExecutors().size();
+ int origRequest = td.getNumWorkers();
+ int slotsRequested = Math.min(totalTasks, origRequest);
+ int slotsUsed = Node.countSlotsUsed(topId, allNodes);
+ int slotsFree = Node.countFreeSlotsAlive(allNodes);
+ //Check to see if we have enough slots before trying to get them
+ int slotsAvailable = 0;
+ if (slotsRequested > slotsFree) {
+ slotsAvailable = NodePool.slotsAvailable(lesserPools);
+ }
+ int slotsToUse = Math.min(slotsRequested - slotsUsed, slotsFree + slotsAvailable);
+ LOG.debug("Slots... requested {} used {} free {} available {} to be used {}",
+ slotsRequested, slotsUsed, slotsFree, slotsAvailable, slotsToUse);
+ if (slotsToUse <= 0) {
+ _cluster.setStatus(topId, "Not Enough Slots Available to Schedule Topology");
+ return 0;
+ }
+ int slotsNeeded = slotsToUse - slotsFree;
+ int numNewNodes = NodePool.getNodeCountIfSlotsWereTaken(slotsNeeded, lesserPools);
+ LOG.debug("Nodes... new {} used {} max {}",
+ numNewNodes, _usedNodes, _maxNodes);
+ if ((numNewNodes + _usedNodes) > _maxNodes) {
+ _cluster.setStatus(topId, "Max Nodes(" + _maxNodes + ") for this user would be exceeded. " +
+ (numNewNodes - (_maxNodes - _usedNodes)) + " more nodes needed to run topology.");
+ return 0;
+ }
+
+ Collection<Node> found = NodePool.takeNodesBySlot(slotsNeeded, lesserPools);
+ _usedNodes += found.size();
+ allNodes.addAll(found);
+ return slotsToUse;
}
-
- Collection<Node> found = NodePool.takeNodesBySlot(slotsNeeded, lesserPools);
- _usedNodes += found.size();
- allNodes.addAll(found);
- return slotsToUse;
- }
- @Override
- public Collection<Node> takeNodes(int nodesNeeded) {
- LOG.debug("Taking {} from {}", nodesNeeded, this);
- HashSet<Node> ret = new HashSet<>();
- for (Entry<String, Set<Node>> entry: _topologyIdToNodes.entrySet()) {
- if (!_isolated.contains(entry.getKey())) {
- Iterator<Node> it = entry.getValue().iterator();
- while (it.hasNext()) {
- if (nodesNeeded <= 0) {
- return ret;
- }
- Node n = it.next();
- it.remove();
- n.freeAllSlots(_cluster);
- ret.add(n);
- nodesNeeded--;
- _usedNodes--;
+ @Override
+ public Collection<Node> takeNodes(int nodesNeeded) {
+ LOG.debug("Taking {} from {}", nodesNeeded, this);
+ HashSet<Node> ret = new HashSet<>();
+ for (Entry<String, Set<Node>> entry : _topologyIdToNodes.entrySet()) {
+ if (!_isolated.contains(entry.getKey())) {
+ Iterator<Node> it = entry.getValue().iterator();
+ while (it.hasNext()) {
+ if (nodesNeeded <= 0) {
+ return ret;
+ }
+ Node n = it.next();
+ it.remove();
+ n.freeAllSlots(_cluster);
+ ret.add(n);
+ nodesNeeded--;
+ _usedNodes--;
+ }
+ }
}
- }
+ return ret;
}
- return ret;
- }
-
- @Override
- public int nodesAvailable() {
- int total = 0;
- for (Entry<String, Set<Node>> entry: _topologyIdToNodes.entrySet()) {
- if (!_isolated.contains(entry.getKey())) {
- total += entry.getValue().size();
- }
+
+ @Override
+ public int nodesAvailable() {
+ int total = 0;
+ for (Entry<String, Set<Node>> entry : _topologyIdToNodes.entrySet()) {
+ if (!_isolated.contains(entry.getKey())) {
+ total += entry.getValue().size();
+ }
+ }
+ return total;
}
- return total;
- }
-
- @Override
- public int slotsAvailable() {
- int total = 0;
- for (Entry<String, Set<Node>> entry: _topologyIdToNodes.entrySet()) {
- if (!_isolated.contains(entry.getKey())) {
- total += Node.countTotalSlotsAlive(entry.getValue());
- }
+
+ @Override
+ public int slotsAvailable() {
+ int total = 0;
+ for (Entry<String, Set<Node>> entry : _topologyIdToNodes.entrySet()) {
+ if (!_isolated.contains(entry.getKey())) {
+ total += Node.countTotalSlotsAlive(entry.getValue());
+ }
+ }
+ return total;
}
- return total;
- }
- @Override
- public Collection<Node> takeNodesBySlots(int slotsNeeded) {
- HashSet<Node> ret = new HashSet<>();
- for (Entry<String, Set<Node>> entry: _topologyIdToNodes.entrySet()) {
- if (!_isolated.contains(entry.getKey())) {
- Iterator<Node> it = entry.getValue().iterator();
- while (it.hasNext()) {
- Node n = it.next();
- if (n.isAlive()) {
- it.remove();
- _usedNodes--;
- n.freeAllSlots(_cluster);
- ret.add(n);
- slotsNeeded -= n.totalSlots();
- if (slotsNeeded <= 0) {
- return ret;
+ @Override
+ public Collection<Node> takeNodesBySlots(int slotsNeeded) {
+ HashSet<Node> ret = new HashSet<>();
+ for (Entry<String, Set<Node>> entry : _topologyIdToNodes.entrySet()) {
+ if (!_isolated.contains(entry.getKey())) {
+ Iterator<Node> it = entry.getValue().iterator();
+ while (it.hasNext()) {
+ Node n = it.next();
+ if (n.isAlive()) {
+ it.remove();
+ _usedNodes--;
+ n.freeAllSlots(_cluster);
+ ret.add(n);
+ slotsNeeded -= n.totalSlots();
+ if (slotsNeeded <= 0) {
+ return ret;
+ }
+ }
+ }
}
- }
}
- }
+ return ret;
}
- return ret;
- }
-
- @Override
- public NodeAndSlotCounts getNodeAndSlotCountIfSlotsWereTaken(int slotsNeeded) {
- int nodesFound = 0;
- int slotsFound = 0;
- for (Entry<String, Set<Node>> entry: _topologyIdToNodes.entrySet()) {
- if (!_isolated.contains(entry.getKey())) {
- for (Node n : entry.getValue()) {
- if (n.isAlive()) {
- nodesFound++;
- int totalSlotsFree = n.totalSlots();
- slotsFound += totalSlotsFree;
- slotsNeeded -= totalSlotsFree;
- if (slotsNeeded <= 0) {
- return new NodeAndSlotCounts(nodesFound, slotsFound);
+
+ @Override
+ public NodeAndSlotCounts getNodeAndSlotCountIfSlotsWereTaken(int slotsNeeded) {
+ int nodesFound = 0;
+ int slotsFound = 0;
+ for (Entry<String, Set<Node>> entry : _topologyIdToNodes.entrySet()) {
+ if (!_isolated.contains(entry.getKey())) {
+ for (Node n : entry.getValue()) {
+ if (n.isAlive()) {
+ nodesFound++;
+ int totalSlotsFree = n.totalSlots();
+ slotsFound += totalSlotsFree;
+ slotsNeeded -= totalSlotsFree;
+ if (slotsNeeded <= 0) {
+ return new NodeAndSlotCounts(nodesFound, slotsFound);
+ }
+ }
+ }
}
- }
}
- }
+ return new NodeAndSlotCounts(nodesFound, slotsFound);
+ }
+
+ @Override
+ public String toString() {
+ return "IsolatedPool... ";
}
- return new NodeAndSlotCounts(nodesFound, slotsFound);
- }
-
- @Override
- public String toString() {
- return "IsolatedPool... ";
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/MultitenantScheduler.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/MultitenantScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/MultitenantScheduler.java
index 65c657e..24c008e 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/MultitenantScheduler.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/MultitenantScheduler.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.multitenant;
@@ -32,94 +26,94 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class MultitenantScheduler implements IScheduler {
- private static final Logger LOG = LoggerFactory.getLogger(MultitenantScheduler.class);
- private Map<String, Object> conf;
- protected IConfigLoader configLoader;
-
- @Override
- public void prepare(Map<String, Object> conf) {
- this.conf = conf;
- configLoader = ConfigLoaderFactoryService.createConfigLoader(conf);
-
- }
-
- /**
- * Load from configLoaders first; if no config available, read from multitenant-scheduler.yaml;
- * if no config available from multitenant-scheduler.yaml, get configs from conf. Only one will be used.
- * @return User pool configs.
- */
- private Map<String, Number> getUserConf() {
- Map<String, Number> ret;
-
- // Try the loader plugin, if configured
- if (configLoader != null) {
- ret = (Map<String, Number>) configLoader.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS);
- if (ret != null) {
- return ret;
- } else {
- LOG.warn("Config loader returned null. Will try to read from multitenant-scheduler.yaml");
- }
- }
+ private static final Logger LOG = LoggerFactory.getLogger(MultitenantScheduler.class);
+ protected IConfigLoader configLoader;
+ private Map<String, Object> conf;
- // If that fails, fall back on the multitenant-scheduler.yaml file
- Map fromFile = Utils.findAndReadConfigFile("multitenant-scheduler.yaml", false);
- ret = (Map<String, Number>)fromFile.get(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS);
- if (ret != null) {
- return ret;
- } else {
- LOG.warn("Reading from multitenant-scheduler.yaml returned null. This could because the file is not available. "
- + "Will load configs from storm configuration");
- }
+ @Override
+ public void prepare(Map<String, Object> conf) {
+ this.conf = conf;
+ configLoader = ConfigLoaderFactoryService.createConfigLoader(conf);
- // If that fails, use config
- ret = (Map<String, Number>) conf.get(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS);
- if (ret == null) {
- return new HashMap<>();
- } else {
- return ret;
}
- }
-
- @Override
- public Map config() {
- return getUserConf();
- }
-
- @Override
- public void schedule(Topologies topologies, Cluster cluster) {
- LOG.debug("Rerunning scheduling...");
- Map<String, Node> nodeIdToNode = Node.getAllNodesFrom(cluster);
-
- Map<String, Number> userConf = getUserConf();
-
- Map<String, IsolatedPool> userPools = new HashMap<>();
- for (Map.Entry<String, Number> entry : userConf.entrySet()) {
- userPools.put(entry.getKey(), new IsolatedPool(entry.getValue().intValue()));
- }
- DefaultPool defaultPool = new DefaultPool();
- FreePool freePool = new FreePool();
-
- freePool.init(cluster, nodeIdToNode);
- for (IsolatedPool pool : userPools.values()) {
- pool.init(cluster, nodeIdToNode);
+
+ /**
+ * Load from configLoaders first; if no config available, read from multitenant-scheduler.yaml;
+ * if no config available from multitenant-scheduler.yaml, get configs from conf. Only one will be used.
+ * @return User pool configs.
+ */
+ private Map<String, Number> getUserConf() {
+ Map<String, Number> ret;
+
+ // Try the loader plugin, if configured
+ if (configLoader != null) {
+ ret = (Map<String, Number>) configLoader.load(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS);
+ if (ret != null) {
+ return ret;
+ } else {
+ LOG.warn("Config loader returned null. Will try to read from multitenant-scheduler.yaml");
+ }
+ }
+
+ // If that fails, fall back on the multitenant-scheduler.yaml file
+ Map fromFile = Utils.findAndReadConfigFile("multitenant-scheduler.yaml", false);
+ ret = (Map<String, Number>) fromFile.get(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS);
+ if (ret != null) {
+ return ret;
+ } else {
+ LOG.warn("Reading from multitenant-scheduler.yaml returned null. This could because the file is not available. "
+ + "Will load configs from storm configuration");
+ }
+
+ // If that fails, use config
+ ret = (Map<String, Number>) conf.get(DaemonConfig.MULTITENANT_SCHEDULER_USER_POOLS);
+ if (ret == null) {
+ return new HashMap<>();
+ } else {
+ return ret;
+ }
}
- defaultPool.init(cluster, nodeIdToNode);
-
- for (TopologyDetails td: topologies.getTopologies()) {
- String user = td.getTopologySubmitter();
- LOG.debug("Found top {} run by user {}",td.getId(), user);
- NodePool pool = userPools.get(user);
- if (pool == null || !pool.canAdd(td)) {
- pool = defaultPool;
- }
- pool.addTopology(td);
+
+ @Override
+ public Map config() {
+ return getUserConf();
}
-
- //Now schedule all of the topologies that need to be scheduled
- for (IsolatedPool pool : userPools.values()) {
- pool.scheduleAsNeeded(freePool, defaultPool);
+
+ @Override
+ public void schedule(Topologies topologies, Cluster cluster) {
+ LOG.debug("Rerunning scheduling...");
+ Map<String, Node> nodeIdToNode = Node.getAllNodesFrom(cluster);
+
+ Map<String, Number> userConf = getUserConf();
+
+ Map<String, IsolatedPool> userPools = new HashMap<>();
+ for (Map.Entry<String, Number> entry : userConf.entrySet()) {
+ userPools.put(entry.getKey(), new IsolatedPool(entry.getValue().intValue()));
+ }
+ DefaultPool defaultPool = new DefaultPool();
+ FreePool freePool = new FreePool();
+
+ freePool.init(cluster, nodeIdToNode);
+ for (IsolatedPool pool : userPools.values()) {
+ pool.init(cluster, nodeIdToNode);
+ }
+ defaultPool.init(cluster, nodeIdToNode);
+
+ for (TopologyDetails td : topologies.getTopologies()) {
+ String user = td.getTopologySubmitter();
+ LOG.debug("Found top {} run by user {}", td.getId(), user);
+ NodePool pool = userPools.get(user);
+ if (pool == null || !pool.canAdd(td)) {
+ pool = defaultPool;
+ }
+ pool.addTopology(td);
+ }
+
+ //Now schedule all of the topologies that need to be scheduled
+ for (IsolatedPool pool : userPools.values()) {
+ pool.scheduleAsNeeded(freePool, defaultPool);
+ }
+ defaultPool.scheduleAsNeeded(freePool);
+ LOG.debug("Scheduling done...");
}
- defaultPool.scheduleAsNeeded(freePool);
- LOG.debug("Scheduling done...");
- }
}
[19/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/LocalCluster.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/LocalCluster.java b/storm-server/src/main/java/org/apache/storm/LocalCluster.java
index 7db41b5..2db0240 100644
--- a/storm-server/src/main/java/org/apache/storm/LocalCluster.java
+++ b/storm-server/src/main/java/org/apache/storm/LocalCluster.java
@@ -74,10 +74,10 @@ import org.apache.storm.generated.RebalanceOptions;
import org.apache.storm.generated.SettableBlobMeta;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.generated.SubmitOptions;
-import org.apache.storm.generated.SupervisorPageInfo;
import org.apache.storm.generated.SupervisorAssignments;
-import org.apache.storm.generated.SupervisorWorkerHeartbeats;
+import org.apache.storm.generated.SupervisorPageInfo;
import org.apache.storm.generated.SupervisorWorkerHeartbeat;
+import org.apache.storm.generated.SupervisorWorkerHeartbeats;
import org.apache.storm.generated.TopologyHistoryInfo;
import org.apache.storm.generated.TopologyInfo;
import org.apache.storm.generated.TopologyPageInfo;
@@ -111,253 +111,21 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * A stand alone storm cluster that runs inside a single process.
- * It is intended to be used for testing. Both internal testing for
- * Apache Storm itself and for people building storm topologies.
- *<p>
- * LocalCluster is an AutoCloseable so if you are using it in tests you can use
- * a try block to be sure it is shut down.
+ * A stand alone storm cluster that runs inside a single process. It is intended to be used for testing. Both internal testing for Apache
+ * Storm itself and for people building storm topologies.
+ * <p>
+ * LocalCluster is an AutoCloseable so if you are using it in tests you can use a try block to be sure it is shut down.
* </p>
- * try (LocalCluster cluster = new LocalCluster()) {
- * // Do some tests
- * }
- * // The cluster has been shut down.
+ * try (LocalCluster cluster = new LocalCluster()) { // Do some tests } // The cluster has been shut down.
*/
public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
+ public static final KillOptions KILL_NOW = new KillOptions();
private static final Logger LOG = LoggerFactory.getLogger(LocalCluster.class);
-
- private static ThriftServer startNimbusDaemon(Map<String, Object> conf, Nimbus nimbus) {
- ThriftServer ret = new ThriftServer(conf, new Processor<>(nimbus), ThriftConnectionType.NIMBUS);
- LOG.info("Starting Nimbus server...");
- new Thread(() -> ret.serve()).start();
- return ret;
- }
-
- /**
- * Simple way to configure a LocalCluster to meet your needs.
- */
- public static class Builder {
- private int supervisors = 2;
- private int portsPerSupervisor = 3;
- private Map<String, Object> daemonConf = new HashMap<>();
- private INimbus inimbus = null;
- private IGroupMappingServiceProvider groupMapper = null;
- private int supervisorSlotPortMin = 1024;
- private boolean nimbusDaemon = false;
- private UnaryOperator<Nimbus> nimbusWrapper = null;
- private BlobStore store = null;
- private TopoCache topoCache = null;
- private IStormClusterState clusterState = null;
- private ILeaderElector leaderElector = null;
- private String trackId = null;
- private boolean simulateTime = false;
-
- /**
- * Set the number of supervisors the cluster should have.
- */
- public Builder withSupervisors(int supervisors) {
- if (supervisors < 0) {
- throw new IllegalArgumentException("supervisors cannot be negative");
- }
- this.supervisors = supervisors;
- return this;
- }
-
- /**
- * Set the number of slots/ports each supervisor should have.
- */
- public Builder withPortsPerSupervisor(int portsPerSupervisor) {
- if (portsPerSupervisor < 0) {
- throw new IllegalArgumentException("supervisor ports cannot be negative");
- }
- this.portsPerSupervisor = portsPerSupervisor;
- return this;
- }
-
- /**
- * Set the base config that the daemons should use.
- */
- public Builder withDaemonConf(Map<String, Object> conf) {
- if (conf != null) {
- this.daemonConf = new HashMap<>(conf);
- }
- return this;
- }
-
- /**
- * Add an single key/value config to the daemon conf.
- */
- public Builder withDaemonConf(String key, Object value) {
- this.daemonConf.put(key, value);
- return this;
- }
-
- /**
- * Override the INimbus instance that nimbus will use.
- */
- public Builder withINimbus(INimbus inimbus) {
- this.inimbus = inimbus;
- return this;
- }
-
- /**
- * Override the code that maps users to groups for authorization.
- */
- public Builder withGroupMapper(IGroupMappingServiceProvider groupMapper) {
- this.groupMapper = groupMapper;
- return this;
- }
-
- /**
- * When assigning ports to worker slots start at minPort.
- */
- public Builder withSupervisorSlotPortMin(Number minPort) {
- int port = 1024;
- if (minPort == null) {
- LOG.warn("Number is null... {}", minPort);
- } else {
- port = minPort.intValue();
- }
- if (port <= 0) {
- throw new IllegalArgumentException("port must be positive");
- }
- this.supervisorSlotPortMin = port;
- return this;
- }
-
- /**
- * Have the local nimbus actually launch a thrift server. This is intended to
- * be used mostly for internal storm testing.
- */
- public Builder withNimbusDaemon() {
- return withNimbusDaemon(true);
- }
-
- /**
- * If nimbusDaemon is true the local nimbus will launch a thrift server. This is intended to
- * be used mostly for internal storm testing.
- */
- public Builder withNimbusDaemon(Boolean nimbusDaemon) {
- if (nimbusDaemon == null) {
- nimbusDaemon = false;
- LOG.warn("nimbusDaemon is null");
- }
- this.nimbusDaemon = nimbusDaemon;
- return this;
- }
-
- /**
- * Turn on simulated time in the cluster. This allows someone to simulate long periods of
- * time for timeouts etc when testing nimbus/supervisors themselves. NOTE: that this only
- * works for code that uses the {@link org.apache.storm.utils.Time} class for time management
- * so it will not work in all cases.
- */
- public Builder withSimulatedTime() {
- return withSimulatedTime(true);
- }
-
- /**
- * Turn on simulated time in the cluster. This allows someone to simulate long periods of
- * time for timeouts etc when testing nimbus/supervisors themselves. NOTE: that this only
- * works for code that uses the {@link org.apache.storm.utils.Time} class for time management
- * so it will not work in all cases.
- */
- public Builder withSimulatedTime(boolean simulateTime) {
- this.simulateTime = simulateTime;
- return this;
- }
-
- /**
- * Before nimbus is created/used call nimbusWrapper on it first and use the
- * result instead. This is intended for internal testing only, and it here to
- * allow a mocking framework to spy on the nimbus class.
- */
- public Builder withNimbusWrapper(UnaryOperator<Nimbus> nimbusWrapper) {
- this.nimbusWrapper = nimbusWrapper;
- return this;
- }
-
- /**
- * Use the following blobstore instead of the one in the config.
- * This is intended mostly for internal testing with Mocks.
- */
- public Builder withBlobStore(BlobStore store) {
- this.store = store;
- return this;
- }
-
- /**
- * Use the following topo cache instead of creating out own.
- * This is intended mostly for internal testing with Mocks.
- */
- public Builder withTopoCache(TopoCache topoCache) {
- this.topoCache = topoCache;
- return this;
- }
- /**
- * Use the following clusterState instead of the one in the config.
- * This is intended mostly for internal testing with Mocks.
- */
- public Builder withClusterState(IStormClusterState clusterState) {
- this.clusterState = clusterState;
- return this;
- }
-
- /**
- * Use the following leaderElector instead of the one in the config.
- * This is intended mostly for internal testing with Mocks.
- */
- public Builder withLeaderElector(ILeaderElector leaderElector) {
- this.leaderElector = leaderElector;
- return this;
- }
-
- /**
- * A tracked cluster can run tracked topologies.
- * See {@link org.apache.storm.testing.TrackedTopology} for more information
- * on tracked topologies.
- * @param trackId an arbitrary unique id that is used to keep track of tracked topologies
- */
- public Builder withTracked(String trackId) {
- this.trackId = trackId;
- return this;
- }
-
- /**
- * A tracked cluster can run tracked topologies.
- * See {@link org.apache.storm.testing.TrackedTopology} for more information
- * on tracked topologies.
- */
- public Builder withTracked() {
- this.trackId = Utils.uuid();
- return this;
- }
-
- /**
- * Builds a new LocalCluster.
- * @return the LocalCluster
- * @throws Exception on any one of many different errors.
- * This is intended for testing so yes it is ugly and throws Exception...
- */
- public LocalCluster build() throws Exception {
- return new LocalCluster(this);
- }
+ static {
+ KILL_NOW.set_wait_secs(0);
}
-
- private static class TrackedStormCommon extends StormCommon {
- private final String id;
- public TrackedStormCommon(String id) {
- this.id = id;
- }
-
- @Override
- public IBolt makeAckerBoltImpl() {
- return new NonRichBoltTracker(new Acker(), id);
- }
- }
-
private final Nimbus nimbus;
//This is very private and does not need to be exposed
private final AtomicInteger portCounter;
@@ -373,27 +141,28 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
private final StormCommonInstaller commonInstaller;
private final SimulatedTime time;
private final NimbusClient.LocalOverride nimbusOverride;
-
/**
* Create a default LocalCluster.
+ *
* @throws Exception on any error
*/
public LocalCluster() throws Exception {
this(new Builder().withDaemonConf(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true));
}
-
+
/**
* Create a LocalCluster that connects to an existing Zookeeper instance.
+ *
* @param zkHost the host for ZK
* @param zkPort the port for ZK
* @throws Exception on any error
*/
public LocalCluster(String zkHost, Long zkPort) throws Exception {
this(new Builder().withDaemonConf(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true)
- .withDaemonConf(Config.STORM_ZOOKEEPER_SERVERS, Arrays.asList(zkHost))
- .withDaemonConf(Config.STORM_ZOOKEEPER_PORT, zkPort));
+ .withDaemonConf(Config.STORM_ZOOKEEPER_SERVERS, Arrays.asList(zkHost))
+ .withDaemonConf(Config.STORM_ZOOKEEPER_PORT, zkPort));
}
-
+
@SuppressWarnings("deprecation")
private LocalCluster(Builder builder) throws Exception {
if (builder.simulateTime) {
@@ -416,7 +185,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
} else {
this.commonInstaller = null;
}
-
+
this.tmpDirs = new ArrayList<>();
this.supervisors = new ArrayList<>();
TmpPath nimbusTmp = new TmpPath();
@@ -428,7 +197,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
conf.put(Config.STORM_CLUSTER_MODE, "local");
conf.put(Config.BLOBSTORE_SUPERUSER, System.getProperty("user.name"));
conf.put(Config.BLOBSTORE_DIR, nimbusTmp.getPath());
-
+
InProcessZookeeper zookeeper = null;
if (!builder.daemonConf.containsKey(Config.STORM_ZOOKEEPER_SERVERS)) {
zookeeper = new InProcessZookeeper();
@@ -438,7 +207,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
this.zookeeper = zookeeper;
conf.putAll(builder.daemonConf);
this.daemonConf = new HashMap<>(conf);
-
+
this.portCounter = new AtomicInteger(builder.supervisorSlotPortMin);
ClusterStateContext cs = new ClusterStateContext(DaemonType.NIMBUS, daemonConf);
this.state = ClusterUtils.mkStateStorage(this.daemonConf, null, cs);
@@ -450,8 +219,8 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
//Set it for nimbus only
conf.put(Config.STORM_LOCAL_DIR, nimbusTmp.getPath());
Nimbus nimbus = new Nimbus(conf, builder.inimbus == null ? new StandaloneINimbus() : builder.inimbus,
- this.getClusterState(), null, builder.store, builder.topoCache, builder.leaderElector,
- builder.groupMapper);
+ this.getClusterState(), null, builder.store, builder.topoCache, builder.leaderElector,
+ builder.groupMapper);
if (builder.nimbusWrapper != null) {
nimbus = builder.nimbusWrapper.apply(nimbus);
}
@@ -464,11 +233,11 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
this.sharedContext = context;
this.thriftServer = builder.nimbusDaemon ? startNimbusDaemon(this.daemonConf, this.nimbus) : null;
-
+
for (int i = 0; i < builder.supervisors; i++) {
addSupervisor(builder.portsPerSupervisor, null, null);
}
-
+
//Wait for a leader to be elected (or topology submission can be rejected)
try {
long timeoutAfter = System.currentTimeMillis() + 10_000;
@@ -479,7 +248,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
Thread.sleep(1);
}
} catch (Exception e) {
- //Ignore any exceptions we might be doing a test for authentication
+ //Ignore any exceptions we might be doing a test for authentication
}
if (thriftServer == null) {
//We don't want to override the client if there is a thrift server up and running, or we would not test any
@@ -496,16 +265,88 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
}
+ private static ThriftServer startNimbusDaemon(Map<String, Object> conf, Nimbus nimbus) {
+ ThriftServer ret = new ThriftServer(conf, new Processor<>(nimbus), ThriftConnectionType.NIMBUS);
+ LOG.info("Starting Nimbus server...");
+ new Thread(() -> ret.serve()).start();
+ return ret;
+ }
+
+ private static boolean areAllWorkersWaiting() {
+ boolean ret = true;
+ for (Shutdownable s : ProcessSimulator.getAllProcessHandles()) {
+ if (s instanceof DaemonCommon) {
+ ret = ret && ((DaemonCommon) s).isWaiting();
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Run c with a local mode cluster overriding the NimbusClient and DRPCClient calls. NOTE local mode override happens by default now
+ * unless netty is turned on for the local cluster.
+ *
+ * @param c the callable to run in this mode
+ * @param ttlSec the number of seconds to let the cluster run after c has completed
+ * @return the result of calling C
+ *
+ * @throws Exception on any Exception.
+ */
+ public static <T> T withLocalModeOverride(Callable<T> c, long ttlSec) throws Exception {
+ LOG.info("\n\n\t\tSTARTING LOCAL MODE CLUSTER\n\n");
+ try (LocalCluster local = new LocalCluster();
+ LocalDRPC drpc = new LocalDRPC();
+ DRPCClient.LocalOverride drpcOverride = new DRPCClient.LocalOverride(drpc)) {
+
+ T ret = c.call();
+ LOG.info("\n\n\t\tRUNNING LOCAL CLUSTER for {} seconds.\n\n", ttlSec);
+ Thread.sleep(ttlSec * 1000);
+
+ LOG.info("\n\n\t\tSTOPPING LOCAL MODE CLUSTER\n\n");
+ return ret;
+ }
+ }
+
+ public static void main(final String[] args) throws Exception {
+ if (args.length < 1) {
+ throw new IllegalArgumentException("No class was specified to run");
+ }
+
+ long ttl = 20;
+ String ttlString = System.getProperty("storm.local.sleeptime", "20");
+ try {
+ ttl = Long.valueOf(ttlString);
+ } catch (NumberFormatException e) {
+ LOG.warn("could not parse the sleep time defaulting to {} seconds", ttl);
+ }
+
+ withLocalModeOverride(() -> {
+ String klass = args[0];
+ String[] newArgs = Arrays.copyOfRange(args, 1, args.length);
+ Class<?> c = Class.forName(klass);
+ Method main = c.getDeclaredMethod("main", String[].class);
+
+ LOG.info("\n\n\t\tRUNNING {} with args {}\n\n", main, Arrays.toString(newArgs));
+ main.invoke(null, (Object) newArgs);
+ return (Void) null;
+ }, ttl);
+
+ //Sometimes external things used with testing don't shut down all the way
+ System.exit(0);
+ }
+
/**
* Checks if Nimbuses have elected a leader.
+ *
* @return boolean
+ *
* @throws AuthorizationException
* @throws TException
*/
private boolean hasLeader() throws AuthorizationException, TException {
ClusterSummary summary = getNimbus().getClusterInfo();
if (summary.is_set_nimbuses()) {
- for (NimbusSummary sum: summary.get_nimbuses()) {
+ for (NimbusSummary sum : summary.get_nimbuses()) {
if (sum.is_isLeader()) {
return true;
}
@@ -520,54 +361,22 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
public Nimbus getNimbus() {
return nimbus;
}
-
+
/**
* @return the base config for the daemons.
*/
public Map<String, Object> getDaemonConf() {
return new HashMap<>(daemonConf);
}
-
- public static final KillOptions KILL_NOW = new KillOptions();
-
- static {
- KILL_NOW.set_wait_secs(0);
- }
-
- /**
- * When running a topology locally, for tests etc. It is helpful to be sure
- * that the topology is dead before the test exits. This is an AutoCloseable
- * topology that not only gives you access to the compiled StormTopology
- * but also will kill the topology when it closes.
- *
- * try (LocalTopology testTopo = cluster.submitTopology("testing", ...)) {
- * // Run Some test
- * }
- * // The topology has been killed
- */
- public class LocalTopology extends StormTopology implements ILocalTopology {
- private static final long serialVersionUID = 6145919776650637748L;
- private final String topoName;
-
- public LocalTopology(String topoName, StormTopology topo) {
- super(topo);
- this.topoName = topoName;
- }
- @Override
- public void close() throws TException {
- killTopologyWithOpts(topoName, KILL_NOW);
- }
- }
-
@Override
public LocalTopology submitTopology(String topologyName, Map<String, Object> conf, StormTopology topology)
- throws TException {
+ throws TException {
if (!Utils.isValidConf(conf)) {
throw new IllegalArgumentException("Topology conf is not json-serializable");
}
getNimbus().submitTopology(topologyName, null, JSONValue.toJSONString(conf), Utils.addVersions(topology));
-
+
ISubmitterHook hook = (ISubmitterHook) Utils.getConfiguredClass(conf, Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN);
if (hook != null) {
TopologyInfo topologyInfo = Utils.getTopologyInfo(topologyName, null, conf);
@@ -581,27 +390,29 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
@Override
- public LocalTopology submitTopologyWithOpts(String topologyName, Map<String, Object> conf, StormTopology topology, SubmitOptions submitOpts)
- throws TException {
+ public LocalTopology submitTopologyWithOpts(String topologyName, Map<String, Object> conf, StormTopology topology,
+ SubmitOptions submitOpts)
+ throws TException {
if (!Utils.isValidConf(conf)) {
throw new IllegalArgumentException("Topology conf is not json-serializable");
}
- getNimbus().submitTopologyWithOpts(topologyName, null, JSONValue.toJSONString(conf), Utils.addVersions(topology), submitOpts);
+ getNimbus().submitTopologyWithOpts(topologyName, null, JSONValue.toJSONString(conf), Utils.addVersions(topology), submitOpts);
return new LocalTopology(topologyName, topology);
}
@Override
public LocalTopology submitTopology(String topologyName, Map<String, Object> conf, TrackedTopology topology)
- throws TException {
+ throws TException {
return submitTopology(topologyName, conf, topology.getTopology());
}
@Override
- public LocalTopology submitTopologyWithOpts(String topologyName, Map<String, Object> conf, TrackedTopology topology, SubmitOptions submitOpts)
- throws TException {
- return submitTopologyWithOpts(topologyName, conf, topology.getTopology(), submitOpts);
+ public LocalTopology submitTopologyWithOpts(String topologyName, Map<String, Object> conf, TrackedTopology topology,
+ SubmitOptions submitOpts)
+ throws TException {
+ return submitTopologyWithOpts(topologyName, conf, topology.getTopology(), submitOpts);
}
-
+
@Override
public void uploadNewCredentials(String topologyName, Credentials creds) throws TException {
getNimbus().uploadNewCredentials(topologyName, creds);
@@ -690,7 +501,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
if (getClusterState() != null) {
getClusterState().disconnect();
}
- for (Supervisor s: supervisors) {
+ for (Supervisor s : supervisors) {
s.shutdownAllWorkers(null, ReadClusterState.THREAD_DUMP_ON_ERROR);
s.close();
}
@@ -700,45 +511,47 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
zookeeper.close();
LOG.info("Done shutting down in process zookeeper");
}
-
- for (TmpPath p: tmpDirs) {
+
+ for (TmpPath p : tmpDirs) {
p.close();
}
-
+
if (this.trackId != null) {
LOG.warn("Clearing tracked metrics for ID {}", this.trackId);
LocalExecutor.clearTrackId();
RegisteredGlobalState.clearState(this.trackId);
}
-
+
if (this.commonInstaller != null) {
this.commonInstaller.close();
}
-
+
if (time != null) {
time.close();
}
}
-
+
/**
* Get a specific Supervisor. This is intended mostly for internal testing.
+ *
* @param id the id of the supervisor
*/
public synchronized Supervisor getSupervisor(String id) {
- for (Supervisor s: supervisors) {
+ for (Supervisor s : supervisors) {
if (id.equals(s.getId())) {
return s;
}
}
return null;
}
-
+
/**
* Kill a specific supervisor. This is intended mostly for internal testing.
+ *
* @param id the id of the supervisor
*/
public synchronized void killSupervisor(String id) {
- for (Iterator<Supervisor> it = supervisors.iterator(); it.hasNext();) {
+ for (Iterator<Supervisor> it = supervisors.iterator(); it.hasNext(); ) {
Supervisor s = it.next();
if (id.equals(s.getId())) {
it.remove();
@@ -748,7 +561,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
}
}
-
+
/**
* Add another supervisor to the topology. This is intended mostly for internal testing.
*/
@@ -758,26 +571,29 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
/**
* Add another supervisor to the topology. This is intended mostly for internal testing.
+ *
* @param ports the number of ports/slots the supervisor should have
*/
public Supervisor addSupervisor(Number ports) throws Exception {
return addSupervisor(ports, null, null);
}
-
+
/**
* Add another supervisor to the topology. This is intended mostly for internal testing.
+ *
* @param ports the number of ports/slots the supervisor should have
- * @param id the id of the new supervisor, so you can find it later.
+ * @param id the id of the new supervisor, so you can find it later.
*/
public Supervisor addSupervisor(Number ports, String id) throws Exception {
return addSupervisor(ports, null, id);
}
-
+
/**
* Add another supervisor to the topology. This is intended mostly for internal testing.
+ *
* @param ports the number of ports/slots the supervisor should have
- * @param conf any config values that should be added/over written in the daemon conf of the cluster.
- * @param id the id of the new supervisor, so you can find it later.
+ * @param conf any config values that should be added/over written in the daemon conf of the cluster.
+ * @param id the id of the new supervisor, so you can find it later.
*/
public synchronized Supervisor addSupervisor(Number ports, Map<String, Object> conf, String id) throws Exception {
if (ports == null) {
@@ -785,19 +601,19 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
TmpPath tmpDir = new TmpPath();
tmpDirs.add(tmpDir);
-
+
List<Integer> portNumbers = new ArrayList<>(ports.intValue());
for (int i = 0; i < ports.intValue(); i++) {
portNumbers.add(portCounter.getAndIncrement());
}
-
+
Map<String, Object> superConf = new HashMap<>(daemonConf);
if (conf != null) {
superConf.putAll(conf);
}
superConf.put(Config.STORM_LOCAL_DIR, tmpDir.getPath());
superConf.put(DaemonConfig.SUPERVISOR_SLOTS_PORTS, portNumbers);
-
+
final String superId = id == null ? Utils.uuid() : id;
ISupervisor isuper = new StandaloneSupervisor() {
@Override
@@ -808,7 +624,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
if (!ConfigUtils.isLocalMode(superConf)) {
throw new IllegalArgumentException("Cannot start server in distrubuted mode!");
}
-
+
Supervisor s = new Supervisor(superConf, sharedContext, isuper);
s.launch();
s.setLocalNimbus(this.nimbus);
@@ -816,51 +632,39 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
supervisors.add(s);
return s;
}
-
+
private boolean areAllSupervisorsWaiting() {
boolean ret = true;
- for (Supervisor s: supervisors) {
+ for (Supervisor s : supervisors) {
ret = ret && s.isWaiting();
}
return ret;
}
-
- private static boolean areAllWorkersWaiting() {
- boolean ret = true;
- for (Shutdownable s: ProcessSimulator.getAllProcessHandles()) {
- if (s instanceof DaemonCommon) {
- ret = ret && ((DaemonCommon)s).isWaiting();
- }
- }
- return ret;
- }
-
+
/**
- * Wait for the cluster to be idle. This is intended to be used with
- * Simulated time and is for internal testing.
+ * Wait for the cluster to be idle. This is intended to be used with Simulated time and is for internal testing.
+ *
* @throws InterruptedException if interrupted while waiting.
- * @throws AssertionError if the cluster did not come to an idle point with
- * a timeout.
+ * @throws AssertionError if the cluster did not come to an idle point with a timeout.
*/
public void waitForIdle() throws InterruptedException {
waitForIdle(Testing.TEST_TIMEOUT_MS);
}
-
+
/**
- * Wait for the cluster to be idle. This is intended to be used with
- * Simulated time and is for internal testing.
+ * Wait for the cluster to be idle. This is intended to be used with Simulated time and is for internal testing.
+ *
* @param timeoutMs the number of ms to wait before throwing an error.
* @throws InterruptedException if interrupted while waiting.
- * @throws AssertionError if the cluster did not come to an idle point with
- * a timeout.
+ * @throws AssertionError if the cluster did not come to an idle point with a timeout.
*/
public void waitForIdle(long timeoutMs) throws InterruptedException {
Random rand = ThreadLocalRandom.current();
//wait until all workers, supervisors, and nimbus is waiting
final long endTime = System.currentTimeMillis() + timeoutMs;
while (!(nimbus.isWaiting() &&
- areAllSupervisorsWaiting() &&
- areAllWorkersWaiting())) {
+ areAllSupervisorsWaiting() &&
+ areAllWorkersWaiting())) {
if (System.currentTimeMillis() >= endTime) {
LOG.info("Cluster was not idle in {} ms", timeoutMs);
LOG.info(Utils.threadDump());
@@ -869,12 +673,12 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
Thread.sleep(rand.nextInt(20));
}
}
-
+
@Override
public void advanceClusterTime(int secs) throws InterruptedException {
advanceClusterTime(secs, 1);
}
-
+
@Override
public void advanceClusterTime(int secs, int incSecs) throws InterruptedException {
for (int amountLeft = secs; amountLeft > 0; amountLeft -= incSecs) {
@@ -888,17 +692,15 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
public IStormClusterState getClusterState() {
return clusterState;
}
-
+
@Override
public String getTrackedId() {
return trackId;
}
- //Nimbus Compatibility
-
@Override
public void submitTopology(String name, String uploadedJarLocation, String jsonConf, StormTopology topology)
- throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, TException {
+ throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, TException {
try {
@SuppressWarnings("unchecked")
Map<String, Object> conf = (Map<String, Object>) JSONValue.parseWithException(jsonConf);
@@ -908,10 +710,12 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
}
}
+ //Nimbus Compatibility
+
@Override
public void submitTopologyWithOpts(String name, String uploadedJarLocation, String jsonConf, StormTopology topology,
- SubmitOptions options)
- throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, TException {
+ SubmitOptions options)
+ throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, TException {
try {
@SuppressWarnings("unchecked")
Map<String, Object> conf = (Map<String, Object>) JSONValue.parseWithException(jsonConf);
@@ -935,7 +739,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
@Override
public void debug(String name, String component, boolean enable, double samplingPercentage)
- throws NotAliveException, AuthorizationException, TException {
+ throws NotAliveException, AuthorizationException, TException {
// TODO Auto-generated method stub
throw new RuntimeException("NOT IMPLEMENTED YET");
}
@@ -948,14 +752,14 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
@Override
public List<ProfileRequest> getComponentPendingProfileActions(String id, String componentId, ProfileAction action)
- throws TException {
+ throws TException {
// TODO Auto-generated method stub
throw new RuntimeException("NOT IMPLEMENTED YET");
}
@Override
public String beginCreateBlob(String key, SettableBlobMeta meta)
- throws AuthorizationException, KeyAlreadyExistsException, TException {
+ throws AuthorizationException, KeyAlreadyExistsException, TException {
throw new RuntimeException("BLOBS NOT SUPPORTED IN LOCAL MODE");
}
@@ -986,13 +790,13 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
@Override
public void setBlobMeta(String key, SettableBlobMeta meta)
- throws AuthorizationException, KeyNotFoundException, TException {
+ throws AuthorizationException, KeyNotFoundException, TException {
throw new KeyNotFoundException("BLOBS NOT SUPPORTED IN LOCAL MODE");
}
@Override
public BeginDownloadResult beginBlobDownload(String key)
- throws AuthorizationException, KeyNotFoundException, TException {
+ throws AuthorizationException, KeyNotFoundException, TException {
throw new KeyNotFoundException("BLOBS NOT SUPPORTED IN LOCAL MODE");
}
@@ -1021,7 +825,7 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
@Override
public int updateBlobReplication(String key, int replication)
- throws AuthorizationException, KeyNotFoundException, TException {
+ throws AuthorizationException, KeyNotFoundException, TException {
throw new KeyNotFoundException("BLOBS NOT SUPPORTED IN LOCAL MODE");
}
@@ -1075,28 +879,28 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
@Override
public TopologyInfo getTopologyInfoWithOpts(String id, GetInfoOptions options)
- throws NotAliveException, AuthorizationException, TException {
+ throws NotAliveException, AuthorizationException, TException {
// TODO Auto-generated method stub
throw new RuntimeException("NOT IMPLEMENTED YET");
}
@Override
public TopologyPageInfo getTopologyPageInfo(String id, String window, boolean is_include_sys)
- throws NotAliveException, AuthorizationException, TException {
+ throws NotAliveException, AuthorizationException, TException {
// TODO Auto-generated method stub
throw new RuntimeException("NOT IMPLEMENTED YET");
}
@Override
public SupervisorPageInfo getSupervisorPageInfo(String id, String host, boolean is_include_sys)
- throws NotAliveException, AuthorizationException, TException {
+ throws NotAliveException, AuthorizationException, TException {
// TODO Auto-generated method stub
throw new RuntimeException("NOT IMPLEMENTED YET");
}
@Override
public ComponentPageInfo getComponentPageInfo(String topology_id, String component_id, String window,
- boolean is_include_sys) throws NotAliveException, AuthorizationException, TException {
+ boolean is_include_sys) throws NotAliveException, AuthorizationException, TException {
// TODO Auto-generated method stub
throw new RuntimeException("NOT IMPLEMENTED YET");
}
@@ -1112,29 +916,6 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
// TODO Auto-generated method stub
throw new RuntimeException("NOT IMPLEMENTED YET");
}
-
- /**
- * Run c with a local mode cluster overriding the NimbusClient and DRPCClient calls.
- * NOTE local mode override happens by default now unless netty is turned on for the local cluster.
- * @param c the callable to run in this mode
- * @param ttlSec the number of seconds to let the cluster run after c has completed
- * @return the result of calling C
- * @throws Exception on any Exception.
- */
- public static <T> T withLocalModeOverride(Callable<T> c, long ttlSec) throws Exception {
- LOG.info("\n\n\t\tSTARTING LOCAL MODE CLUSTER\n\n");
- try (LocalCluster local = new LocalCluster();
- LocalDRPC drpc = new LocalDRPC();
- DRPCClient.LocalOverride drpcOverride = new DRPCClient.LocalOverride(drpc)) {
-
- T ret = c.call();
- LOG.info("\n\n\t\tRUNNING LOCAL CLUSTER for {} seconds.\n\n", ttlSec);
- Thread.sleep(ttlSec * 1000);
-
- LOG.info("\n\n\t\tSTOPPING LOCAL MODE CLUSTER\n\n");
- return ret;
- }
- }
@Override
public List<OwnerResourceSummary> getOwnerResourceSummaries(String owner) throws AuthorizationException, TException {
@@ -1161,31 +942,243 @@ public class LocalCluster implements ILocalClusterTrackedTopologyAware, Iface {
getNimbus().processWorkerMetrics(metrics);
}
- public static void main(final String [] args) throws Exception {
- if (args.length < 1) {
- throw new IllegalArgumentException("No class was specified to run");
+ /**
+ * Simple way to configure a LocalCluster to meet your needs.
+ */
+ public static class Builder {
+ private int supervisors = 2;
+ private int portsPerSupervisor = 3;
+ private Map<String, Object> daemonConf = new HashMap<>();
+ private INimbus inimbus = null;
+ private IGroupMappingServiceProvider groupMapper = null;
+ private int supervisorSlotPortMin = 1024;
+ private boolean nimbusDaemon = false;
+ private UnaryOperator<Nimbus> nimbusWrapper = null;
+ private BlobStore store = null;
+ private TopoCache topoCache = null;
+ private IStormClusterState clusterState = null;
+ private ILeaderElector leaderElector = null;
+ private String trackId = null;
+ private boolean simulateTime = false;
+
+ /**
+ * Set the number of supervisors the cluster should have.
+ */
+ public Builder withSupervisors(int supervisors) {
+ if (supervisors < 0) {
+ throw new IllegalArgumentException("supervisors cannot be negative");
+ }
+ this.supervisors = supervisors;
+ return this;
}
-
- long ttl = 20;
- String ttlString = System.getProperty("storm.local.sleeptime", "20");
- try {
- ttl = Long.valueOf(ttlString);
- } catch (NumberFormatException e) {
- LOG.warn("could not parse the sleep time defaulting to {} seconds", ttl);
+
+ /**
+ * Set the number of slots/ports each supervisor should have.
+ */
+ public Builder withPortsPerSupervisor(int portsPerSupervisor) {
+ if (portsPerSupervisor < 0) {
+ throw new IllegalArgumentException("supervisor ports cannot be negative");
+ }
+ this.portsPerSupervisor = portsPerSupervisor;
+ return this;
+ }
+
+ /**
+ * Set the base config that the daemons should use.
+ */
+ public Builder withDaemonConf(Map<String, Object> conf) {
+ if (conf != null) {
+ this.daemonConf = new HashMap<>(conf);
+ }
+ return this;
+ }
+
+ /**
+ * Add an single key/value config to the daemon conf.
+ */
+ public Builder withDaemonConf(String key, Object value) {
+ this.daemonConf.put(key, value);
+ return this;
+ }
+
+ /**
+ * Override the INimbus instance that nimbus will use.
+ */
+ public Builder withINimbus(INimbus inimbus) {
+ this.inimbus = inimbus;
+ return this;
+ }
+
+ /**
+ * Override the code that maps users to groups for authorization.
+ */
+ public Builder withGroupMapper(IGroupMappingServiceProvider groupMapper) {
+ this.groupMapper = groupMapper;
+ return this;
+ }
+
+ /**
+ * When assigning ports to worker slots start at minPort.
+ */
+ public Builder withSupervisorSlotPortMin(Number minPort) {
+ int port = 1024;
+ if (minPort == null) {
+ LOG.warn("Number is null... {}", minPort);
+ } else {
+ port = minPort.intValue();
+ }
+ if (port <= 0) {
+ throw new IllegalArgumentException("port must be positive");
+ }
+ this.supervisorSlotPortMin = port;
+ return this;
+ }
+
+ /**
+ * Have the local nimbus actually launch a thrift server. This is intended to be used mostly for internal storm testing.
+ */
+ public Builder withNimbusDaemon() {
+ return withNimbusDaemon(true);
+ }
+
+ /**
+ * If nimbusDaemon is true the local nimbus will launch a thrift server. This is intended to be used mostly for internal storm
+ * testing.
+ */
+ public Builder withNimbusDaemon(Boolean nimbusDaemon) {
+ if (nimbusDaemon == null) {
+ nimbusDaemon = false;
+ LOG.warn("nimbusDaemon is null");
+ }
+ this.nimbusDaemon = nimbusDaemon;
+ return this;
+ }
+
+ /**
+ * Turn on simulated time in the cluster. This allows someone to simulate long periods of time for timeouts etc when testing
+ * nimbus/supervisors themselves. NOTE: that this only works for code that uses the {@link org.apache.storm.utils.Time} class for
+ * time management so it will not work in all cases.
+ */
+ public Builder withSimulatedTime() {
+ return withSimulatedTime(true);
+ }
+
+ /**
+ * Turn on simulated time in the cluster. This allows someone to simulate long periods of time for timeouts etc when testing
+ * nimbus/supervisors themselves. NOTE: that this only works for code that uses the {@link org.apache.storm.utils.Time} class for
+ * time management so it will not work in all cases.
+ */
+ public Builder withSimulatedTime(boolean simulateTime) {
+ this.simulateTime = simulateTime;
+ return this;
+ }
+
+ /**
+ * Before nimbus is created/used call nimbusWrapper on it first and use the result instead. This is intended for internal testing
+ * only, and it here to allow a mocking framework to spy on the nimbus class.
+ */
+ public Builder withNimbusWrapper(UnaryOperator<Nimbus> nimbusWrapper) {
+ this.nimbusWrapper = nimbusWrapper;
+ return this;
+ }
+
+ /**
+ * Use the following blobstore instead of the one in the config. This is intended mostly for internal testing with Mocks.
+ */
+ public Builder withBlobStore(BlobStore store) {
+ this.store = store;
+ return this;
+ }
+
+ /**
+ * Use the following topo cache instead of creating out own. This is intended mostly for internal testing with Mocks.
+ */
+ public Builder withTopoCache(TopoCache topoCache) {
+ this.topoCache = topoCache;
+ return this;
+ }
+
+ /**
+ * Use the following clusterState instead of the one in the config. This is intended mostly for internal testing with Mocks.
+ */
+ public Builder withClusterState(IStormClusterState clusterState) {
+ this.clusterState = clusterState;
+ return this;
+ }
+
+ /**
+ * Use the following leaderElector instead of the one in the config. This is intended mostly for internal testing with Mocks.
+ */
+ public Builder withLeaderElector(ILeaderElector leaderElector) {
+ this.leaderElector = leaderElector;
+ return this;
+ }
+
+ /**
+ * A tracked cluster can run tracked topologies. See {@link org.apache.storm.testing.TrackedTopology} for more information on
+ * tracked topologies.
+ *
+ * @param trackId an arbitrary unique id that is used to keep track of tracked topologies
+ */
+ public Builder withTracked(String trackId) {
+ this.trackId = trackId;
+ return this;
+ }
+
+ /**
+ * A tracked cluster can run tracked topologies. See {@link org.apache.storm.testing.TrackedTopology} for more information on
+ * tracked topologies.
+ */
+ public Builder withTracked() {
+ this.trackId = Utils.uuid();
+ return this;
+ }
+
+ /**
+ * Builds a new LocalCluster.
+ *
+ * @return the LocalCluster
+ *
+ * @throws Exception on any one of many different errors. This is intended for testing so yes it is ugly and throws Exception...
+ */
+ public LocalCluster build() throws Exception {
+ return new LocalCluster(this);
+ }
+ }
+
+ private static class TrackedStormCommon extends StormCommon {
+
+ private final String id;
+
+ public TrackedStormCommon(String id) {
+ this.id = id;
+ }
+
+ @Override
+ public IBolt makeAckerBoltImpl() {
+ return new NonRichBoltTracker(new Acker(), id);
+ }
+ }
+
+ /**
+ * When running a topology locally, for tests etc. It is helpful to be sure that the topology is dead before the test exits. This is
+ * an AutoCloseable topology that not only gives you access to the compiled StormTopology but also will kill the topology when it
+ * closes.
+ *
+ * try (LocalTopology testTopo = cluster.submitTopology("testing", ...)) { // Run Some test } // The topology has been killed
+ */
+ public class LocalTopology extends StormTopology implements ILocalTopology {
+ private static final long serialVersionUID = 6145919776650637748L;
+ private final String topoName;
+
+ public LocalTopology(String topoName, StormTopology topo) {
+ super(topo);
+ this.topoName = topoName;
+ }
+
+ @Override
+ public void close() throws TException {
+ killTopologyWithOpts(topoName, KILL_NOW);
}
-
- withLocalModeOverride(() -> {
- String klass = args[0];
- String [] newArgs = Arrays.copyOfRange(args, 1, args.length);
- Class<?> c = Class.forName(klass);
- Method main = c.getDeclaredMethod("main", String[].class);
-
- LOG.info("\n\n\t\tRUNNING {} with args {}\n\n", main, Arrays.toString(newArgs));
- main.invoke(null, (Object)newArgs);
- return (Void)null;
- }, ttl);
-
- //Sometimes external things used with testing don't shut down all the way
- System.exit(0);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/LocalDRPC.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/LocalDRPC.java b/storm-server/src/main/java/org/apache/storm/LocalDRPC.java
index bd60b62..ef257ed 100644
--- a/storm-server/src/main/java/org/apache/storm/LocalDRPC.java
+++ b/storm-server/src/main/java/org/apache/storm/LocalDRPC.java
@@ -15,10 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm;
import java.util.Map;
-
import org.apache.storm.daemon.drpc.DRPC;
import org.apache.storm.daemon.drpc.DRPCThrift;
import org.apache.storm.generated.AuthorizationException;
@@ -30,10 +30,8 @@ import org.apache.thrift.TException;
/**
* A Local way to test DRPC
- *
- * try (LocalDRPC drpc = new LocalDRPC()) {
- * // Do tests
- * }
+ *
+ * try (LocalDRPC drpc = new LocalDRPC()) { // Do tests }
*/
public class LocalDRPC implements ILocalDRPC {
@@ -65,7 +63,7 @@ public class LocalDRPC implements ILocalDRPC {
public void failRequest(String id) throws AuthorizationException, TException {
drpc.failRequest(id, null);
}
-
+
@Override
public void failRequestV2(String id, DRPCExecutionException e) throws AuthorizationException, TException {
@@ -82,7 +80,7 @@ public class LocalDRPC implements ILocalDRPC {
ServiceRegistry.unregisterService(this.serviceId);
drpc.close();
}
-
+
@Override
public void shutdown() {
close();
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/ProcessSimulator.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/ProcessSimulator.java b/storm-server/src/main/java/org/apache/storm/ProcessSimulator.java
index 1388e07..f2e81cf 100644
--- a/storm-server/src/main/java/org/apache/storm/ProcessSimulator.java
+++ b/storm-server/src/main/java/org/apache/storm/ProcessSimulator.java
@@ -1,29 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm;
-import org.apache.storm.daemon.Shutdownable;
-import org.apache.storm.utils.Utils;
import java.nio.channels.ClosedByInterruptException;
import java.util.Collection;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-
+import org.apache.storm.daemon.Shutdownable;
+import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,13 +26,13 @@ import org.slf4j.LoggerFactory;
* in place of actual processes (in cluster mode).
*/
public class ProcessSimulator {
+ protected static ConcurrentHashMap<String, Shutdownable> processMap = new ConcurrentHashMap<String, Shutdownable>();
private static Logger LOG = LoggerFactory.getLogger(ProcessSimulator.class);
private static Object lock = new Object();
- protected static ConcurrentHashMap<String, Shutdownable> processMap = new ConcurrentHashMap<String, Shutdownable>();
/**
* Register a process' handle
- *
+ *
* @param pid
* @param shutdownable
*/
@@ -48,7 +42,7 @@ public class ProcessSimulator {
/**
* Get all process handles
- *
+ *
* @return
*/
public static Collection<Shutdownable> getAllProcessHandles() {
@@ -57,7 +51,7 @@ public class ProcessSimulator {
/**
* Kill a process
- *
+ *
* @param pid
*/
public static void killProcess(String pid) {
@@ -85,7 +79,7 @@ public class ProcessSimulator {
LOG.warn("process {} not killed (Ignoring InterruptedException)", pid, e);
} else if (Utils.exceptionCauseIsInstanceOf(ClosedByInterruptException.class, e)) {
LOG.warn("process {} not killed (Ignoring ClosedByInterruptException)", pid, e);
- } else if (e instanceof RuntimeException){
+ } else if (e instanceof RuntimeException) {
throw e;
} else {
//TODO once everything is in java this should not be possible any more
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/Testing.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/Testing.java b/storm-server/src/main/java/org/apache/storm/Testing.java
index c391926..5a8495b 100644
--- a/storm-server/src/main/java/org/apache/storm/Testing.java
+++ b/storm-server/src/main/java/org/apache/storm/Testing.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm;
@@ -29,7 +23,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
import java.util.stream.Collectors;
-
import org.apache.storm.cluster.IStormClusterState;
import org.apache.storm.generated.Bolt;
import org.apache.storm.generated.GlobalStreamId;
@@ -66,13 +59,14 @@ import org.slf4j.LoggerFactory;
* A utility that helps with testing topologies, Bolts and Spouts.
*/
public class Testing {
- private static final Logger LOG = LoggerFactory.getLogger(Testing.class);
/**
- * The default amount of wall time should be spent waiting for
+ * The default amount of wall time should be spent waiting for
* specific conditions to happen. Default is 10 seconds unless
* the environment variable STORM_TEST_TIMEOUT_MS is set.
*/
public static final int TEST_TIMEOUT_MS;
+ private static final Logger LOG = LoggerFactory.getLogger(Testing.class);
+
static {
int timeout = 10_000;
try {
@@ -82,14 +76,7 @@ public class Testing {
}
TEST_TIMEOUT_MS = timeout;
}
-
- /**
- * Simply produces a boolean to see if a specific state is true or false.
- */
- public static interface Condition {
- public boolean exec();
- }
-
+
/**
* Continue to execute body repeatedly until condition is true or TEST_TIMEOUT_MS has
* passed
@@ -100,7 +87,7 @@ public class Testing {
public static void whileTimeout(Condition condition, Runnable body) {
whileTimeout(TEST_TIMEOUT_MS, condition, body);
}
-
+
/**
* Continue to execute body repeatedly until condition is true or TEST_TIMEOUT_MS has
* passed
@@ -124,17 +111,17 @@ public class Testing {
}
LOG.debug("Condition met {}", condition);
}
-
+
/**
* Convenience method for data.stream.allMatch(pred)
*/
public static <T> boolean isEvery(Collection<T> data, Predicate<T> pred) {
return data.stream().allMatch(pred);
}
-
+
/**
* Run with simulated time
- * @deprecated use ```
+ * @deprecated use ```
* try (Time.SimulatedTime time = new Time.SimulatedTime()) {
* ...
* }
@@ -147,15 +134,15 @@ public class Testing {
code.run();
}
}
-
+
private static LocalCluster cluster(MkClusterParam param, boolean simulated) throws Exception {
return cluster(param, null, simulated);
}
-
+
private static LocalCluster cluster(MkClusterParam param) throws Exception {
return cluster(param, null, false);
}
-
+
private static LocalCluster cluster(MkClusterParam param, String id, boolean simulated) throws Exception {
Integer supervisors = param.getSupervisors();
if (supervisors == null) {
@@ -170,18 +157,18 @@ public class Testing {
conf = new HashMap<>();
}
return new LocalCluster.Builder()
- .withSupervisors(supervisors)
- .withPortsPerSupervisor(ports)
- .withDaemonConf(conf)
- .withNimbusDaemon(param.isNimbusDaemon())
- .withTracked(id)
- .withSimulatedTime(simulated)
- .build();
+ .withSupervisors(supervisors)
+ .withPortsPerSupervisor(ports)
+ .withDaemonConf(conf)
+ .withNimbusDaemon(param.isNimbusDaemon())
+ .withTracked(id)
+ .withSimulatedTime(simulated)
+ .build();
}
-
+
/**
* Run with a local cluster
- * @deprecated use ```
+ * @deprecated use ```
* try (LocalCluster cluster = new LocalCluster()) {
* ...
* }
@@ -192,10 +179,10 @@ public class Testing {
public static void withLocalCluster(TestJob code) {
withLocalCluster(new MkClusterParam(), code);
}
-
+
/**
* Run with a local cluster
- * @deprecated use ```
+ * @deprecated use ```
* try (LocalCluster cluster = new LocalCluster.Builder()....build()) {
* ...
* }
@@ -211,10 +198,10 @@ public class Testing {
throw new RuntimeException(e);
}
}
-
+
/**
* Run with a local cluster
- * @deprecated use ```
+ * @deprecated use ```
* try (LocalCluster cluster = new LocalCluster.Builder()....build()) {
* ...
* }
@@ -235,21 +222,21 @@ public class Testing {
Boolean nimbusDaemon = (Boolean) clusterConf.getOrDefault("nimbus-daemon", false);
try {
return new LocalCluster.Builder()
- .withSupervisors(supervisors.intValue())
- .withDaemonConf(conf)
- .withPortsPerSupervisor(ports.intValue())
- .withINimbus(inimbus)
- .withSupervisorSlotPortMin(portMin)
- .withNimbusDaemon(nimbusDaemon)
- .build();
+ .withSupervisors(supervisors.intValue())
+ .withDaemonConf(conf)
+ .withPortsPerSupervisor(ports.intValue())
+ .withINimbus(inimbus)
+ .withSupervisorSlotPortMin(portMin)
+ .withNimbusDaemon(nimbusDaemon)
+ .build();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
-
+
/**
* Run with a local cluster
- * @deprecated use ```
+ * @deprecated use ```
* try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().build()) {
* ...
* }
@@ -260,10 +247,10 @@ public class Testing {
public static void withSimulatedTimeLocalCluster(TestJob code) {
withSimulatedTimeLocalCluster(new MkClusterParam(), code);
}
-
+
/**
* Run with a local cluster
- * @deprecated use ```
+ * @deprecated use ```
* try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime()....build()) {
* ...
* }
@@ -279,10 +266,10 @@ public class Testing {
throw new RuntimeException(e);
}
}
-
+
/**
* Run with a local cluster
- * @deprecated use ```
+ * @deprecated use ```
* try (LocalCluster cluster = new LocalCluster.Builder().withTracked().build()) {
* ...
* }
@@ -293,7 +280,7 @@ public class Testing {
public static void withTrackedCluster(TestJob code) {
withTrackedCluster(new MkClusterParam(), code);
}
-
+
/**
* In a tracked topology some metrics are tracked. This provides a way to get those metrics.
* This is intended mostly for internal testing.
@@ -305,12 +292,12 @@ public class Testing {
@Deprecated
public static int globalAmt(String id, String key) {
LOG.warn("Reading tracked metrics for ID {}", id);
- return ((ConcurrentHashMap<String, AtomicInteger>)RegisteredGlobalState.getState(id)).get(key).get();
+ return ((ConcurrentHashMap<String, AtomicInteger>) RegisteredGlobalState.getState(id)).get(key).get();
}
-
+
/**
* Run with a local tracked cluster
- * @deprecated use ```
+ * @deprecated use ```
* try (LocalCluster cluster = new LocalCluster.Builder().withTracked()....build()) {
* ...
* }
@@ -326,25 +313,7 @@ public class Testing {
throw new RuntimeException(e);
}
}
-
- /**
- * A topology that has all messages captured and can be read later on.
- * This is intended mostly for internal testing.
- * @param <T> the topology (tracked or regular)
- */
- public static final class CapturedTopology<T> {
- public final T topology;
- /**
- * a Bolt that will hold all of the captured data.
- */
- public final TupleCaptureBolt capturer;
-
- public CapturedTopology(T topology, TupleCaptureBolt capturer) {
- this.topology = topology;
- this.capturer = capturer;
- }
- }
-
+
/**
* Track and capture a topology.
* This is intended mostly for internal testing.
@@ -353,7 +322,7 @@ public class Testing {
CapturedTopology<StormTopology> captured = captureTopology(topology);
return new CapturedTopology<>(new TrackedTopology(captured.topology, cluster), captured.capturer);
}
-
+
/**
* Rewrites a topology so that all the tuples flowing through it are captured
* @param topology the topology to rewrite
@@ -362,7 +331,7 @@ public class Testing {
*/
public static CapturedTopology<StormTopology> captureTopology(StormTopology topology) {
topology = topology.deepCopy(); //Don't modify the original
-
+
TupleCaptureBolt capturer = new TupleCaptureBolt();
Map<GlobalStreamId, Grouping> captureBoltInputs = new HashMap<>();
for (Map.Entry<String, SpoutSpec> spoutEntry : topology.get_spouts().entrySet()) {
@@ -377,7 +346,7 @@ public class Testing {
}
}
}
-
+
for (Entry<String, Bolt> boltEntry : topology.get_bolts().entrySet()) {
String id = boltEntry.getKey();
for (Entry<String, StreamInfo> streamEntry : boltEntry.getValue().get_common().get_streams().entrySet()) {
@@ -391,35 +360,36 @@ public class Testing {
}
}
topology.put_to_bolts(Utils.uuid(), new Bolt(Thrift.serializeComponentObject(capturer),
- Thrift.prepareComponentCommon(captureBoltInputs, new HashMap<>(), null)));
+ Thrift.prepareComponentCommon(captureBoltInputs, new HashMap<>(), null)));
return new CapturedTopology<>(topology, capturer);
}
-
+
/**
* Run a topology to completion capturing all of the messages that are emitted. This only works when all of the spouts are
* instances of {@link org.apache.storm.testing.CompletableSpout}
* @param cluster the cluster to submit the topology to
* @param topology the topology itself
- * @return a map of the component to the list of tuples it emitted.
+ * @return a map of the component to the list of tuples it emitted.
* @throws InterruptedException
* @throws TException on any error from nimbus.
*/
- public static Map<String, List<FixedTuple>> completeTopology(ILocalCluster cluster, StormTopology topology) throws InterruptedException, TException {
+ public static Map<String, List<FixedTuple>> completeTopology(ILocalCluster cluster, StormTopology topology) throws InterruptedException,
+ TException {
return completeTopology(cluster, topology, new CompleteTopologyParam());
}
-
+
/**
* Run a topology to completion capturing all of the messages that are emitted. This only works when all of the spouts are
* instances of {@link org.apache.storm.testing.CompletableSpout} or are overwritten by MockedSources in param
* @param cluster the cluster to submit the topology to
* @param topology the topology itself
* @param param parameters to describe how to complete a topology.
- * @return a map of the component to the list of tuples it emitted.
+ * @return a map of the component to the list of tuples it emitted.
* @throws InterruptedException
* @throws TException on any error from nimbus.
*/
public static Map<String, List<FixedTuple>> completeTopology(ILocalCluster cluster, StormTopology topology,
- CompleteTopologyParam param) throws TException, InterruptedException {
+ CompleteTopologyParam param) throws TException, InterruptedException {
Map<String, List<FixedTuple>> ret = null;
IStormClusterState state = cluster.getClusterState();
CapturedTopology<StormTopology> capTopo = captureTopology(topology);
@@ -432,30 +402,31 @@ public class Testing {
Map<String, SpoutSpec> spouts = topology.get_spouts();
MockedSources ms = param.getMockedSources();
if (ms != null) {
- for (Entry<String, List<FixedTuple>> mocked: ms.getData().entrySet()) {
+ for (Entry<String, List<FixedTuple>> mocked : ms.getData().entrySet()) {
FixedTupleSpout newSpout = new FixedTupleSpout(mocked.getValue());
spouts.get(mocked.getKey()).set_spout_object(Thrift.serializeComponentObject(newSpout));
}
}
List<Object> spoutObjects = spouts.values().stream().
- map((spec) -> Thrift.deserializeComponentObject(spec.get_spout_object())).collect(Collectors.toList());
-
- for (Object o: spoutObjects) {
+ map((spec) -> Thrift.deserializeComponentObject(spec.get_spout_object())).collect(Collectors.toList());
+
+ for (Object o : spoutObjects) {
if (!(o instanceof CompletableSpout)) {
- throw new RuntimeException("Cannot complete topology unless every spout is a CompletableSpout (or mocked to be); failed by " + o);
+ throw new RuntimeException(
+ "Cannot complete topology unless every spout is a CompletableSpout (or mocked to be); failed by " + o);
}
}
-
- for (Object spout: spoutObjects) {
- ((CompletableSpout)spout).startup();
+
+ for (Object spout : spoutObjects) {
+ ((CompletableSpout) spout).startup();
}
-
+
cluster.submitTopology(topoName, param.getStormConf(), topology);
-
+
if (Time.isSimulating()) {
cluster.advanceClusterTime(11);
}
-
+
String topoId = state.getTopoId(topoName).get();
//Give the topology time to come up without using it to wait for the spouts to complete
simulateWait(cluster);
@@ -464,41 +435,41 @@ public class Testing {
timeoutMs = TEST_TIMEOUT_MS;
}
whileTimeout(timeoutMs,
- () -> !isEvery(spoutObjects, (o) -> ((CompletableSpout)o).isExhausted()),
- () -> {
- try {
- simulateWait(cluster);
- } catch (Exception e) {
- throw new RuntimeException();
- }
- });
+ () -> !isEvery(spoutObjects, (o) -> ((CompletableSpout) o).isExhausted()),
+ () -> {
+ try {
+ simulateWait(cluster);
+ } catch (Exception e) {
+ throw new RuntimeException();
+ }
+ });
KillOptions killOpts = new KillOptions();
killOpts.set_wait_secs(0);
cluster.killTopologyWithOpts(topoName, killOpts);
-
+
whileTimeout(timeoutMs,
- () -> state.assignmentInfo(topoId, null) != null,
- () -> {
- try {
- simulateWait(cluster);
- } catch (Exception e) {
- throw new RuntimeException();
- }
- });
-
+ () -> state.assignmentInfo(topoId, null) != null,
+ () -> {
+ try {
+ simulateWait(cluster);
+ } catch (Exception e) {
+ throw new RuntimeException();
+ }
+ });
+
if (param.getCleanupState()) {
for (Object o : spoutObjects) {
- ((CompletableSpout)o).clean();
+ ((CompletableSpout) o).clean();
}
ret = capTopo.capturer.getAndRemoveResults();
} else {
ret = capTopo.capturer.getAndClearResults();
}
-
+
return ret;
}
-
+
/**
* If using simulated time simulate waiting for 10 seconds. This is intended for internal testing only.
*/
@@ -508,7 +479,7 @@ public class Testing {
Thread.sleep(100);
}
}
-
+
/**
* Get all of the tuples from a given component on the default stream
* @param results the results of running a completed topology
@@ -530,7 +501,7 @@ public class Testing {
List<List<Object>> ret = new ArrayList<>();
List<FixedTuple> streamResult = results.get(componentId);
if (streamResult != null) {
- for (FixedTuple tuple: streamResult) {
+ for (FixedTuple tuple : streamResult) {
if (streamId.equals(tuple.stream)) {
ret.add(tuple.values);
}
@@ -538,7 +509,7 @@ public class Testing {
}
return ret;
}
-
+
/**
* Create a tracked topology.
* @deprecated use {@link org.apache.storm.testing.TrackedTopology} directly.
@@ -547,63 +518,63 @@ public class Testing {
public static TrackedTopology mkTrackedTopology(ILocalCluster cluster, StormTopology topology) {
return new TrackedTopology(topology, cluster);
}
-
+
/**
* Simulated time wait for a tracked topology. This is intended for internal testing
*/
public static void trackedWait(CapturedTopology<TrackedTopology> topo) {
topo.topology.trackedWait();
}
-
+
/**
* Simulated time wait for a tracked topology. This is intended for internal testing
*/
public static void trackedWait(CapturedTopology<TrackedTopology> topo, Integer amt) {
topo.topology.trackedWait(amt);
}
-
+
/**
* Simulated time wait for a tracked topology. This is intended for internal testing
*/
public static void trackedWait(CapturedTopology<TrackedTopology> topo, Integer amt, Integer timeoutMs) {
topo.topology.trackedWait(amt, timeoutMs);
}
-
+
/**
* Simulated time wait for a tracked topology. This is intended for internal testing
*/
public static void trackedWait(TrackedTopology topo) {
topo.trackedWait();
}
-
+
/**
* Simulated time wait for a tracked topology. This is intended for internal testing
*/
public static void trackedWait(TrackedTopology topo, Integer amt) {
topo.trackedWait(amt);
}
-
+
/**
* Simulated time wait for a tracked topology. This is intended for internal testing
*/
public static void trackedWait(TrackedTopology topo, Integer amt, Integer timeoutMs) {
topo.trackedWait(amt, timeoutMs);
}
-
+
/**
* Simulated time wait for a cluster. This is intended for internal testing
*/
public static void advanceClusterTime(ILocalCluster cluster, Integer secs) throws InterruptedException {
advanceClusterTime(cluster, secs, 1);
}
-
+
/**
* Simulated time wait for a cluster. This is intended for internal testing
*/
public static void advanceClusterTime(ILocalCluster cluster, Integer secs, Integer step) throws InterruptedException {
cluster.advanceClusterTime(secs, step);
}
-
+
/**
* Count how many times each element appears in the Collection
* @param c a collection of values
@@ -611,7 +582,7 @@ public class Testing {
*/
public static <T> Map<T, Integer> multiset(Collection<T> c) {
Map<T, Integer> ret = new HashMap<T, Integer>();
- for (T t: c) {
+ for (T t : c) {
Integer i = ret.get(t);
if (i == null) {
i = new Integer(0);
@@ -621,40 +592,40 @@ public class Testing {
}
return ret;
}
-
+
private static void printRec(Object o, String prefix) {
if (o instanceof Collection) {
- LOG.info("{} {} ({}) [",prefix,o, o.getClass());
- for (Object sub: (Collection)o) {
+ LOG.info("{} {} ({}) [", prefix, o, o.getClass());
+ for (Object sub : (Collection) o) {
printRec(sub, prefix + " ");
}
- LOG.info("{} ]",prefix);
+ LOG.info("{} ]", prefix);
} else if (o instanceof Map) {
- Map<?,?> m = (Map<?,?>)o;
- LOG.info("{} {} ({}) {",prefix,o, o.getClass());
- for (Map.Entry<?, ?> entry: m.entrySet()) {
+ Map<?, ?> m = (Map<?, ?>) o;
+ LOG.info("{} {} ({}) {", prefix, o, o.getClass());
+ for (Map.Entry<?, ?> entry : m.entrySet()) {
printRec(entry.getKey(), prefix + " ");
LOG.info("{} ->", prefix);
printRec(entry.getValue(), prefix + " ");
}
- LOG.info("{} }",prefix);
+ LOG.info("{} }", prefix);
} else {
LOG.info("{} {} ({})", prefix, o, o.getClass());
}
}
-
+
/**
* Check if two collections are equivalent ignoring the order of elements.
*/
public static <T> boolean multiseteq(Collection<T> a, Collection<T> b) {
boolean ret = multiset(a).equals(multiset(b));
- if (! ret) {
+ if (!ret) {
printRec(multiset(a), "MS-A:");
printRec(multiset(b), "MS-B:");
}
return ret;
}
-
+
/**
* Create a {@link org.apache.storm.tuple.Tuple} for use with testing
* @param values the values to appear in the tuple
@@ -662,7 +633,7 @@ public class Testing {
public static Tuple testTuple(List<Object> values) {
return testTuple(values, new MkTupleParam());
}
-
+
/**
* Create a {@link org.apache.storm.tuple.Tuple} for use with testing
* @param values the values to appear in the tuple
@@ -673,19 +644,19 @@ public class Testing {
if (stream == null) {
stream = Utils.DEFAULT_STREAM_ID;
}
-
+
String component = param.getComponent();
if (component == null) {
component = "component";
}
-
+
int task = 1;
-
+
List<String> fields = param.getFields();
if (fields == null) {
fields = new ArrayList<>(values.size());
for (int i = 1; i <= values.size(); i++) {
- fields.add("field"+i);
+ fields.add("field" + i);
}
}
@@ -696,23 +667,48 @@ public class Testing {
streamToFields.put(stream, new Fields(fields));
compToStreamToFields.put(component, streamToFields);
- TopologyContext context= new TopologyContext(null,
- ConfigUtils.readStormConfig(),
- taskToComp,
- null,
- compToStreamToFields,
- null,
- "test-storm-id",
- null,
- null,
- 1,
- null,
- null,
- new HashMap<>(),
- new HashMap<>(),
- new HashMap<>(),
- new HashMap<>(),
- new AtomicBoolean(false));
+ TopologyContext context = new TopologyContext(null,
+ ConfigUtils.readStormConfig(),
+ taskToComp,
+ null,
+ compToStreamToFields,
+ null,
+ "test-storm-id",
+ null,
+ null,
+ 1,
+ null,
+ null,
+ new HashMap<>(),
+ new HashMap<>(),
+ new HashMap<>(),
+ new HashMap<>(),
+ new AtomicBoolean(false));
return new TupleImpl(context, values, component, 1, stream);
}
+
+ /**
+ * Simply produces a boolean to see if a specific state is true or false.
+ */
+ public static interface Condition {
+ public boolean exec();
+ }
+
+ /**
+ * A topology that has all messages captured and can be read later on.
+ * This is intended mostly for internal testing.
+ * @param <T> the topology (tracked or regular)
+ */
+ public static final class CapturedTopology<T> {
+ public final T topology;
+ /**
+ * a Bolt that will hold all of the captured data.
+ */
+ public final TupleCaptureBolt capturer;
+
+ public CapturedTopology(T topology, TupleCaptureBolt capturer) {
+ this.topology = topology;
+ this.capturer = capturer;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/blobstore/BlobKeySequenceInfo.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/BlobKeySequenceInfo.java b/storm-server/src/main/java/org/apache/storm/blobstore/BlobKeySequenceInfo.java
index 8a01f1e..e50575a 100644
--- a/storm-server/src/main/java/org/apache/storm/blobstore/BlobKeySequenceInfo.java
+++ b/storm-server/src/main/java/org/apache/storm/blobstore/BlobKeySequenceInfo.java
@@ -1,40 +1,35 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.blobstore;
public class BlobKeySequenceInfo {
private String nimbusHostPort;
private String sequenceNumber;
- public void setNimbusHostPort(String nimbusHostPort) {
- this.nimbusHostPort = nimbusHostPort;
- }
-
- public void setSequenceNumber(String sequenceNumber) {
- this.sequenceNumber = sequenceNumber;
- }
-
public String getNimbusHostPort() {
return nimbusHostPort;
}
+ public void setNimbusHostPort(String nimbusHostPort) {
+ this.nimbusHostPort = nimbusHostPort;
+ }
+
public String getSequenceNumber() {
return sequenceNumber;
}
+ public void setSequenceNumber(String sequenceNumber) {
+ this.sequenceNumber = sequenceNumber;
+ }
+
}
[54/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-hive
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/test/java/org/apache/storm/hive/bolt/TestHiveBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/test/java/org/apache/storm/hive/bolt/TestHiveBolt.java b/external/storm-hive/src/test/java/org/apache/storm/hive/bolt/TestHiveBolt.java
index a18b91e..35815fd 100644
--- a/external/storm-hive/src/test/java/org/apache/storm/hive/bolt/TestHiveBolt.java
+++ b/external/storm-hive/src/test/java/org/apache/storm/hive/bolt/TestHiveBolt.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hive.bolt;
@@ -21,7 +15,23 @@ package org.apache.storm.hive.bolt;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hive.hcatalog.streaming.HiveEndPoint;
import org.apache.storm.Config;
+import org.apache.storm.hive.bolt.mapper.DelimitedRecordHiveMapper;
+import org.apache.storm.hive.bolt.mapper.JsonRecordHiveMapper;
+import org.apache.storm.hive.common.HiveOptions;
import org.apache.storm.hive.common.HiveWriter;
import org.apache.storm.task.GeneralTopologyContext;
import org.apache.storm.task.OutputCollector;
@@ -31,41 +41,21 @@ import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.TupleImpl;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.MockTupleHelpers;
-
-import org.apache.storm.hive.common.HiveOptions;
-import org.apache.storm.hive.bolt.mapper.DelimitedRecordHiveMapper;
-import org.apache.storm.hive.bolt.mapper.JsonRecordHiveMapper;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
-import static org.mockito.Mockito.*;
-
-import org.junit.Assert;
-
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.HashSet;
-import java.text.SimpleDateFormat;
-
-
-import org.apache.hive.hcatalog.streaming.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
public class TestHiveBolt {
final static String dbName = "testdb";
@@ -74,87 +64,23 @@ public class TestHiveBolt {
final static String tblName1 = "test_table1";
final static String PART1_NAME = "city";
final static String PART2_NAME = "state";
- final static String[] partNames = {PART1_NAME, PART2_NAME};
- final String partitionVals = "sunnyvale,ca";
+ final static String[] partNames = { PART1_NAME, PART2_NAME };
private static final String COL1 = "id";
private static final String COL2 = "msg";
- final String[] colNames = {COL1, COL2};
- final String[] colNames1 = {COL2, COL1};
- private String[] colTypes = {serdeConstants.INT_TYPE_NAME, serdeConstants.STRING_TYPE_NAME};
- private final HiveConf conf;
+ private static final Logger LOG = LoggerFactory.getLogger(HiveBolt.class);
+ final String partitionVals = "sunnyvale,ca";
+ final String[] colNames = { COL1, COL2 };
+ final String[] colNames1 = { COL2, COL1 };
final String metaStoreURI;
+ private final HiveConf conf;
+ private String[] colTypes = { serdeConstants.INT_TYPE_NAME, serdeConstants.STRING_TYPE_NAME };
private Config config = new Config();
private TestingHiveBolt bolt;
- private ObjectMapper objectMapper = new ObjectMapper();;
-
+ ;
+ private ObjectMapper objectMapper = new ObjectMapper();
@Mock
private OutputCollector collector;
- private static final Logger LOG = LoggerFactory.getLogger(HiveBolt.class);
-
- private static class TestingHiveBolt extends HiveBolt {
-
- protected Map<List<String>, List<byte[]>> partitionValuesToWrittenRecords = new HashMap<>();
-
- public TestingHiveBolt(HiveOptions options) {
- super(options);
- }
-
- @Override
- HiveWriter getOrCreateWriter(final HiveEndPoint endPoint)
- throws HiveWriter.ConnectFailure, InterruptedException {
- HiveWriter writer = allWriters.get(endPoint);
- if (writer == null) {
- // always provide mocked HiveWriter
- writer = Mockito.mock(HiveWriter.class);
- try {
- Mockito.doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- Object[] arguments = invocation.getArguments();
- List<String> partitionVals = endPoint.partitionVals;
- List<byte[]> writtenRecords = partitionValuesToWrittenRecords.get(partitionVals);
- if (writtenRecords == null) {
- writtenRecords = new ArrayList<>();
- partitionValuesToWrittenRecords.put(partitionVals, writtenRecords);
- }
- writtenRecords.add((byte[]) arguments[0]);
- return null;
- }
- }).when(writer).write(any(byte[].class));
- } catch (Exception exc) {
- throw new RuntimeException(exc);
- }
- }
- return writer;
- }
-
- public Map<List<String>, List<byte[]>> getPartitionValuesToWrittenRecords() {
- return partitionValuesToWrittenRecords;
- }
-
- public List<byte[]> getRecordWritten(List<String> partitionValues) {
- return partitionValuesToWrittenRecords.get(partitionValues);
- }
- }
-
- private static class FlushFailureHiveBolt extends TestingHiveBolt {
-
- public FlushFailureHiveBolt(HiveOptions options) {
- super(options);
- }
-
- @Override
- void flushAllWriters(boolean rollToNext) throws HiveWriter.CommitFailure, HiveWriter.TxnBatchFailure, HiveWriter.TxnFailure,
- InterruptedException {
- if (rollToNext) {
- throw new InterruptedException();
- } else {
- super.flushAllWriters(false);
- }
- }
- }
-
public TestHiveBolt() throws Exception {
//metaStoreURI = "jdbc:derby:;databaseName="+System.getProperty("java.io.tmpdir") +"metastore_db;create=true";
metaStoreURI = null;
@@ -172,13 +98,13 @@ public class TestHiveBolt {
@Test
public void testWithByteArrayIdandMessage()
- throws Exception {
+ throws Exception {
DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper()
- .withColumnFields(new Fields(colNames))
- .withPartitionFields(new Fields(partNames));
+ .withColumnFields(new Fields(colNames))
+ .withPartitionFields(new Fields(partNames));
HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper)
- .withTxnsPerBatch(2)
- .withBatchSize(2);
+ .withTxnsPerBatch(2)
+ .withBatchSize(2);
bolt = new TestingHiveBolt(hiveOptions);
bolt.prepare(config, null, collector);
@@ -206,14 +132,13 @@ public class TestHiveBolt {
bolt.cleanup();
}
-
@Test
public void testWithoutPartitions()
- throws Exception {
+ throws Exception {
DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper()
- .withColumnFields(new Fields(colNames));
+ .withColumnFields(new Fields(colNames));
HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName1, tblName1, mapper)
- .withTxnsPerBatch(2).withBatchSize(2).withAutoCreatePartitions(false);
+ .withTxnsPerBatch(2).withBatchSize(2).withAutoCreatePartitions(false);
bolt = new TestingHiveBolt(hiveOptions);
bolt.prepare(config, null, collector);
@@ -245,15 +170,15 @@ public class TestHiveBolt {
@Test
public void testWithTimeformat()
- throws Exception {
+ throws Exception {
String timeFormat = "yyyy/MM/dd";
DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper()
- .withColumnFields(new Fields(colNames))
- .withTimeAsPartitionField(timeFormat);
+ .withColumnFields(new Fields(colNames))
+ .withTimeAsPartitionField(timeFormat);
HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName1, tblName1, mapper)
- .withTxnsPerBatch(2)
- .withBatchSize(1)
- .withMaxOpenConnections(1);
+ .withTxnsPerBatch(2)
+ .withBatchSize(1)
+ .withMaxOpenConnections(1);
bolt = new TestingHiveBolt(hiveOptions);
bolt.prepare(config, null, collector);
@@ -292,13 +217,13 @@ public class TestHiveBolt {
@Test
public void testData()
- throws Exception {
+ throws Exception {
DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper()
- .withColumnFields(new Fields(colNames))
- .withPartitionFields(new Fields(partNames));
+ .withColumnFields(new Fields(colNames))
+ .withPartitionFields(new Fields(partNames));
HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper)
- .withTxnsPerBatch(2)
- .withBatchSize(1);
+ .withTxnsPerBatch(2)
+ .withBatchSize(1);
bolt = new TestingHiveBolt(hiveOptions);
bolt.prepare(config, null, new OutputCollector(collector));
@@ -327,15 +252,15 @@ public class TestHiveBolt {
@Test
public void testJsonWriter()
- throws Exception {
+ throws Exception {
// json record doesn't need columns to be in the same order
// as table in hive.
JsonRecordHiveMapper mapper = new JsonRecordHiveMapper()
- .withColumnFields(new Fields(colNames1))
- .withPartitionFields(new Fields(partNames));
+ .withColumnFields(new Fields(colNames1))
+ .withPartitionFields(new Fields(partNames));
HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper)
- .withTxnsPerBatch(2)
- .withBatchSize(1);
+ .withTxnsPerBatch(2)
+ .withBatchSize(1);
bolt = new TestingHiveBolt(hiveOptions);
bolt.prepare(config, null, collector);
@@ -373,11 +298,11 @@ public class TestHiveBolt {
@Test
public void testNoAcksUntilFlushed() {
JsonRecordHiveMapper mapper = new JsonRecordHiveMapper()
- .withColumnFields(new Fields(colNames1))
- .withPartitionFields(new Fields(partNames));
+ .withColumnFields(new Fields(colNames1))
+ .withPartitionFields(new Fields(partNames));
HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper)
- .withTxnsPerBatch(2)
- .withBatchSize(2);
+ .withTxnsPerBatch(2)
+ .withBatchSize(2);
bolt = new TestingHiveBolt(hiveOptions);
bolt.prepare(config, null, new OutputCollector(collector));
@@ -397,11 +322,11 @@ public class TestHiveBolt {
@Test
public void testNoAcksIfFlushFails() throws Exception {
JsonRecordHiveMapper mapper = new JsonRecordHiveMapper()
- .withColumnFields(new Fields(colNames1))
- .withPartitionFields(new Fields(partNames));
+ .withColumnFields(new Fields(colNames1))
+ .withPartitionFields(new Fields(partNames));
HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper)
- .withTxnsPerBatch(2)
- .withBatchSize(2);
+ .withTxnsPerBatch(2)
+ .withBatchSize(2);
HiveBolt failingBolt = new FlushFailureHiveBolt(hiveOptions);
@@ -422,11 +347,11 @@ public class TestHiveBolt {
@Test
public void testTickTuple() {
JsonRecordHiveMapper mapper = new JsonRecordHiveMapper()
- .withColumnFields(new Fields(colNames1))
- .withPartitionFields(new Fields(partNames));
+ .withColumnFields(new Fields(colNames1))
+ .withPartitionFields(new Fields(partNames));
HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper)
- .withTxnsPerBatch(2)
- .withBatchSize(2);
+ .withTxnsPerBatch(2)
+ .withBatchSize(2);
bolt = new TestingHiveBolt(hiveOptions);
bolt.prepare(config, null, new OutputCollector(collector));
@@ -452,11 +377,11 @@ public class TestHiveBolt {
@Test
public void testNoTickEmptyBatches() throws Exception {
JsonRecordHiveMapper mapper = new JsonRecordHiveMapper()
- .withColumnFields(new Fields(colNames1))
- .withPartitionFields(new Fields(partNames));
+ .withColumnFields(new Fields(colNames1))
+ .withPartitionFields(new Fields(partNames));
HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper)
- .withTxnsPerBatch(2)
- .withBatchSize(2);
+ .withTxnsPerBatch(2)
+ .withBatchSize(2);
bolt = new TestingHiveBolt(hiveOptions);
bolt.prepare(config, null, new OutputCollector(collector));
@@ -471,13 +396,13 @@ public class TestHiveBolt {
@Test
public void testMultiPartitionTuples()
- throws Exception {
+ throws Exception {
DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper()
- .withColumnFields(new Fields(colNames))
- .withPartitionFields(new Fields(partNames));
+ .withColumnFields(new Fields(colNames))
+ .withPartitionFields(new Fields(partNames));
HiveOptions hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper)
- .withTxnsPerBatch(10)
- .withBatchSize(10);
+ .withTxnsPerBatch(10)
+ .withBatchSize(10);
bolt = new TestingHiveBolt(hiveOptions);
bolt.prepare(config, null, new OutputCollector(collector));
@@ -517,7 +442,7 @@ public class TestHiveBolt {
private Tuple generateTestTuple(Object id, Object msg, Object city, Object state) {
TopologyBuilder builder = new TopologyBuilder();
GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(),
- new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
+ new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
@Override
public Fields getComponentOutputFields(String componentId, String streamId) {
return new Fields("id", "msg", "city", "state");
@@ -535,4 +460,67 @@ public class TestHiveBolt {
return builder.toString().getBytes();
}
+ private static class TestingHiveBolt extends HiveBolt {
+
+ protected Map<List<String>, List<byte[]>> partitionValuesToWrittenRecords = new HashMap<>();
+
+ public TestingHiveBolt(HiveOptions options) {
+ super(options);
+ }
+
+ @Override
+ HiveWriter getOrCreateWriter(final HiveEndPoint endPoint)
+ throws HiveWriter.ConnectFailure, InterruptedException {
+ HiveWriter writer = allWriters.get(endPoint);
+ if (writer == null) {
+ // always provide mocked HiveWriter
+ writer = Mockito.mock(HiveWriter.class);
+ try {
+ Mockito.doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] arguments = invocation.getArguments();
+ List<String> partitionVals = endPoint.partitionVals;
+ List<byte[]> writtenRecords = partitionValuesToWrittenRecords.get(partitionVals);
+ if (writtenRecords == null) {
+ writtenRecords = new ArrayList<>();
+ partitionValuesToWrittenRecords.put(partitionVals, writtenRecords);
+ }
+ writtenRecords.add((byte[]) arguments[0]);
+ return null;
+ }
+ }).when(writer).write(any(byte[].class));
+ } catch (Exception exc) {
+ throw new RuntimeException(exc);
+ }
+ }
+ return writer;
+ }
+
+ public Map<List<String>, List<byte[]>> getPartitionValuesToWrittenRecords() {
+ return partitionValuesToWrittenRecords;
+ }
+
+ public List<byte[]> getRecordWritten(List<String> partitionValues) {
+ return partitionValuesToWrittenRecords.get(partitionValues);
+ }
+ }
+
+ private static class FlushFailureHiveBolt extends TestingHiveBolt {
+
+ public FlushFailureHiveBolt(HiveOptions options) {
+ super(options);
+ }
+
+ @Override
+ void flushAllWriters(boolean rollToNext) throws HiveWriter.CommitFailure, HiveWriter.TxnBatchFailure, HiveWriter.TxnFailure,
+ InterruptedException {
+ if (rollToNext) {
+ throw new InterruptedException();
+ } else {
+ super.flushAllWriters(false);
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/test/java/org/apache/storm/hive/common/TestHiveWriter.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/test/java/org/apache/storm/hive/common/TestHiveWriter.java b/external/storm-hive/src/test/java/org/apache/storm/hive/common/TestHiveWriter.java
index 450342b..03d3383 100644
--- a/external/storm-hive/src/test/java/org/apache/storm/hive/common/TestHiveWriter.java
+++ b/external/storm-hive/src/test/java/org/apache/storm/hive/common/TestHiveWriter.java
@@ -1,140 +1,76 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hive.common;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import junit.framework.Assert;
-import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.hcatalog.streaming.HiveEndPoint;
import org.apache.hive.hcatalog.streaming.RecordWriter;
+import org.apache.hive.hcatalog.streaming.SerializationError;
import org.apache.hive.hcatalog.streaming.StreamingConnection;
import org.apache.hive.hcatalog.streaming.StreamingException;
-import org.apache.hive.hcatalog.streaming.SerializationError;
import org.apache.hive.hcatalog.streaming.TransactionBatch;
+import org.apache.storm.Config;
+import org.apache.storm.hive.bolt.HiveSetupUtil;
import org.apache.storm.hive.bolt.mapper.DelimitedRecordHiveMapper;
import org.apache.storm.hive.bolt.mapper.HiveMapper;
-import org.apache.storm.hive.bolt.HiveSetupUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import org.apache.storm.Config;
import org.apache.storm.task.GeneralTopologyContext;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.TupleImpl;
import org.apache.storm.tuple.Values;
-import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.ArrayList;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.HashMap;
-
public class TestHiveWriter {
- final static String dbName = "testdb";
- final static String tblName = "test_table2";
-
public static final String PART1_NAME = "city";
public static final String PART2_NAME = "state";
public static final String[] partNames = { PART1_NAME, PART2_NAME };
- final String[] partitionVals = {"sunnyvale","ca"};
- final String[] colNames = {"id","msg"};
+ final static String dbName = "testdb";
+ final static String tblName = "test_table2";
+ final String[] partitionVals = { "sunnyvale", "ca" };
+ final String[] colNames = { "id", "msg" };
private final int port;
private final String metaStoreURI;
private final HiveConf conf;
- private ExecutorService callTimeoutPool;
- int timeout = 10000; // msec
- UserGroupInformation ugi = null;
-
@Rule
public TemporaryFolder dbFolder = new TemporaryFolder();
-
- private static class TestingHiveWriter extends HiveWriter {
-
- private StreamingConnection mockedStreamingConn;
- private TransactionBatch mockedTxBatch;
-
- public TestingHiveWriter(HiveEndPoint endPoint, int txnsPerBatch, boolean autoCreatePartitions, long callTimeout, ExecutorService callTimeoutPool, HiveMapper mapper, UserGroupInformation ugi, boolean tokenAuthEnabled) throws InterruptedException, ConnectFailure {
- super(endPoint, txnsPerBatch, autoCreatePartitions, callTimeout, callTimeoutPool, mapper, ugi, tokenAuthEnabled);
- }
-
- @Override
- synchronized StreamingConnection newConnection(UserGroupInformation ugi, boolean tokenAuthEnabled) throws InterruptedException, ConnectFailure {
- if (mockedStreamingConn == null) {
- mockedStreamingConn = Mockito.mock(StreamingConnection.class);
- mockedTxBatch = Mockito.mock(TransactionBatch.class);
-
- try {
- Mockito.when(mockedStreamingConn.fetchTransactionBatch(Mockito.anyInt(), Mockito.any(RecordWriter.class)))
- .thenReturn(mockedTxBatch);
- } catch (StreamingException e) {
- throw new RuntimeException(e);
- }
- }
-
- return mockedStreamingConn;
- }
-
- public TransactionBatch getMockedTxBatch() {
- return mockedTxBatch;
- }
- }
-
- private static class MockedDelemiteredRecordHiveMapper extends DelimitedRecordHiveMapper {
- private final RecordWriter mockedRecordWriter;
-
- public MockedDelemiteredRecordHiveMapper() {
- this.mockedRecordWriter = Mockito.mock(RecordWriter.class);
- }
-
- @Override
- public RecordWriter createRecordWriter(HiveEndPoint endPoint) throws StreamingException, IOException, ClassNotFoundException {
- return mockedRecordWriter;
- }
-
- public RecordWriter getMockedRecordWriter() {
- return mockedRecordWriter;
- }
- }
+ int timeout = 10000; // msec
+ UserGroupInformation ugi = null;
+ private ExecutorService callTimeoutPool;
public TestHiveWriter() throws Exception {
port = 9083;
metaStoreURI = null;
int callTimeoutPoolSize = 1;
callTimeoutPool = Executors.newFixedThreadPool(callTimeoutPoolSize,
- new ThreadFactoryBuilder().setNameFormat("hiveWriterTest").build());
+ new ThreadFactoryBuilder().setNameFormat("hiveWriterTest").build());
// 1) Start metastore
conf = HiveSetupUtil.getHiveConf();
TxnDbUtil.setConfValues(conf);
- if(metaStoreURI!=null) {
+ if (metaStoreURI != null) {
conf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreURI);
}
}
@@ -142,23 +78,23 @@ public class TestHiveWriter {
@Test
public void testInstantiate() throws Exception {
DelimitedRecordHiveMapper mapper = new MockedDelemiteredRecordHiveMapper()
- .withColumnFields(new Fields(colNames))
- .withPartitionFields(new Fields(partNames));
+ .withColumnFields(new Fields(colNames))
+ .withPartitionFields(new Fields(partNames));
HiveEndPoint endPoint = new HiveEndPoint(metaStoreURI, dbName, tblName, Arrays.asList(partitionVals));
TestingHiveWriter writer = new TestingHiveWriter(endPoint, 10, true, timeout
- ,callTimeoutPool, mapper, ugi, false);
+ , callTimeoutPool, mapper, ugi, false);
writer.close();
}
@Test
public void testWriteBasic() throws Exception {
DelimitedRecordHiveMapper mapper = new MockedDelemiteredRecordHiveMapper()
- .withColumnFields(new Fields(colNames))
- .withPartitionFields(new Fields(partNames));
+ .withColumnFields(new Fields(colNames))
+ .withPartitionFields(new Fields(partNames));
HiveEndPoint endPoint = new HiveEndPoint(metaStoreURI, dbName, tblName, Arrays.asList(partitionVals));
TestingHiveWriter writer = new TestingHiveWriter(endPoint, 10, true, timeout
- , callTimeoutPool, mapper, ugi, false);
- writeTuples(writer,mapper,3);
+ , callTimeoutPool, mapper, ugi, false);
+ writeTuples(writer, mapper, 3);
writer.flush(false);
writer.close();
Mockito.verify(writer.getMockedTxBatch(), Mockito.times(3)).write(Mockito.any(byte[].class));
@@ -167,15 +103,15 @@ public class TestHiveWriter {
@Test
public void testWriteMultiFlush() throws Exception {
DelimitedRecordHiveMapper mapper = new MockedDelemiteredRecordHiveMapper()
- .withColumnFields(new Fields(colNames))
- .withPartitionFields(new Fields(partNames));
+ .withColumnFields(new Fields(colNames))
+ .withPartitionFields(new Fields(partNames));
HiveEndPoint endPoint = new HiveEndPoint(metaStoreURI, dbName, tblName, Arrays.asList(partitionVals));
TestingHiveWriter writer = new TestingHiveWriter(endPoint, 10, true, timeout
- , callTimeoutPool, mapper, ugi, false);
- Tuple tuple = generateTestTuple("1","abc");
+ , callTimeoutPool, mapper, ugi, false);
+ Tuple tuple = generateTestTuple("1", "abc");
writer.write(mapper.mapRecord(tuple));
- tuple = generateTestTuple("2","def");
+ tuple = generateTestTuple("2", "def");
writer.write(mapper.mapRecord(tuple));
Assert.assertEquals(writer.getTotalRecords(), 2);
Mockito.verify(writer.getMockedTxBatch(), Mockito.times(2)).write(Mockito.any(byte[].class));
@@ -184,11 +120,11 @@ public class TestHiveWriter {
Assert.assertEquals(writer.getTotalRecords(), 0);
Mockito.verify(writer.getMockedTxBatch(), Mockito.atLeastOnce()).commit();
- tuple = generateTestTuple("3","ghi");
+ tuple = generateTestTuple("3", "ghi");
writer.write(mapper.mapRecord(tuple));
writer.flush(true);
- tuple = generateTestTuple("4","klm");
+ tuple = generateTestTuple("4", "klm");
writer.write(mapper.mapRecord(tuple));
writer.flush(true);
writer.close();
@@ -198,7 +134,7 @@ public class TestHiveWriter {
private Tuple generateTestTuple(Object id, Object msg) {
TopologyBuilder builder = new TopologyBuilder();
GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(),
- new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
+ new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
@Override
public Fields getComponentOutputFields(String componentId, String streamId) {
return new Fields("id", "msg");
@@ -208,13 +144,64 @@ public class TestHiveWriter {
}
private void writeTuples(HiveWriter writer, HiveMapper mapper, int count)
- throws HiveWriter.WriteFailure, InterruptedException, SerializationError {
+ throws HiveWriter.WriteFailure, InterruptedException, SerializationError {
Integer id = 100;
String msg = "test-123";
for (int i = 1; i <= count; i++) {
- Tuple tuple = generateTestTuple(id,msg);
+ Tuple tuple = generateTestTuple(id, msg);
writer.write(mapper.mapRecord(tuple));
}
}
+ private static class TestingHiveWriter extends HiveWriter {
+
+ private StreamingConnection mockedStreamingConn;
+ private TransactionBatch mockedTxBatch;
+
+ public TestingHiveWriter(HiveEndPoint endPoint, int txnsPerBatch, boolean autoCreatePartitions, long callTimeout,
+ ExecutorService callTimeoutPool, HiveMapper mapper, UserGroupInformation ugi,
+ boolean tokenAuthEnabled) throws InterruptedException, ConnectFailure {
+ super(endPoint, txnsPerBatch, autoCreatePartitions, callTimeout, callTimeoutPool, mapper, ugi, tokenAuthEnabled);
+ }
+
+ @Override
+ synchronized StreamingConnection newConnection(UserGroupInformation ugi, boolean tokenAuthEnabled) throws InterruptedException,
+ ConnectFailure {
+ if (mockedStreamingConn == null) {
+ mockedStreamingConn = Mockito.mock(StreamingConnection.class);
+ mockedTxBatch = Mockito.mock(TransactionBatch.class);
+
+ try {
+ Mockito.when(mockedStreamingConn.fetchTransactionBatch(Mockito.anyInt(), Mockito.any(RecordWriter.class)))
+ .thenReturn(mockedTxBatch);
+ } catch (StreamingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ return mockedStreamingConn;
+ }
+
+ public TransactionBatch getMockedTxBatch() {
+ return mockedTxBatch;
+ }
+ }
+
+ private static class MockedDelemiteredRecordHiveMapper extends DelimitedRecordHiveMapper {
+ private final RecordWriter mockedRecordWriter;
+
+ public MockedDelemiteredRecordHiveMapper() {
+ this.mockedRecordWriter = Mockito.mock(RecordWriter.class);
+ }
+
+ @Override
+ public RecordWriter createRecordWriter(HiveEndPoint endPoint) throws StreamingException, IOException, ClassNotFoundException {
+ return mockedRecordWriter;
+ }
+
+ public RecordWriter getMockedRecordWriter() {
+ return mockedRecordWriter;
+ }
+ }
+
}
\ No newline at end of file
[57/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-jms
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-jms
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/95602b1b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/95602b1b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/95602b1b
Branch: refs/heads/master
Commit: 95602b1be7493c33b7dc8c3a8cb6e406b59e907d
Parents: 224633d
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 23:23:30 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 02:32:41 2018 -0400
----------------------------------------------------------------------
external/storm-jms/pom.xml | 2 +-
.../apache/storm/jms/JmsMessageProducer.java | 21 +-
.../java/org/apache/storm/jms/JmsProvider.java | 20 +-
.../org/apache/storm/jms/JmsTupleProducer.java | 21 +-
.../java/org/apache/storm/jms/bolt/JmsBolt.java | 38 ++-
.../apache/storm/jms/spout/JmsMessageID.java | 29 +--
.../org/apache/storm/jms/spout/JmsSpout.java | 216 +++++++++--------
.../org/apache/storm/jms/trident/JmsBatch.java | 19 +-
.../org/apache/storm/jms/trident/JmsState.java | 103 ++++----
.../storm/jms/trident/JmsStateFactory.java | 22 +-
.../apache/storm/jms/trident/JmsUpdater.java | 26 +-
.../storm/jms/trident/TridentJmsSpout.java | 237 +++++++++----------
.../apache/storm/jms/spout/JmsSpoutTest.java | 37 ++-
.../apache/storm/jms/spout/MockJmsProvider.java | 21 +-
.../jms/spout/MockSpoutOutputCollector.java | 6 +-
.../storm/jms/spout/MockTupleProducer.java | 2 +-
16 files changed, 382 insertions(+), 438 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-jms/pom.xml b/external/storm-jms/pom.xml
index 81e7f62..c7bcc51 100644
--- a/external/storm-jms/pom.xml
+++ b/external/storm-jms/pom.xml
@@ -94,7 +94,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>235</maxAllowedViolations>
+ <maxAllowedViolations>63</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsMessageProducer.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsMessageProducer.java b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsMessageProducer.java
index 4932929..671cdd9 100644
--- a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsMessageProducer.java
+++ b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsMessageProducer.java
@@ -1,28 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jms;
import java.io.Serializable;
-
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.Session;
-
import org.apache.storm.tuple.ITuple;
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsProvider.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsProvider.java b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsProvider.java
index d976326..b8dde44 100644
--- a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsProvider.java
+++ b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsProvider.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jms;
import java.io.Serializable;
-
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsTupleProducer.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsTupleProducer.java b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsTupleProducer.java
index 0bbb3a0..4457f5a 100644
--- a/external/storm-jms/src/main/java/org/apache/storm/jms/JmsTupleProducer.java
+++ b/external/storm-jms/src/main/java/org/apache/storm/jms/JmsTupleProducer.java
@@ -1,27 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jms;
import java.io.Serializable;
-
import javax.jms.JMSException;
import javax.jms.Message;
-
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Values;
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/main/java/org/apache/storm/jms/bolt/JmsBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/bolt/JmsBolt.java b/external/storm-jms/src/main/java/org/apache/storm/jms/bolt/JmsBolt.java
index 9b3b614..0b461a1 100644
--- a/external/storm-jms/src/main/java/org/apache/storm/jms/bolt/JmsBolt.java
+++ b/external/storm-jms/src/main/java/org/apache/storm/jms/bolt/JmsBolt.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jms.bolt;
import java.util.Map;
-
import javax.jms.Connection;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
@@ -26,19 +20,15 @@ import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.MessageProducer;
import javax.jms.Session;
-
-import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.jms.JmsMessageProducer;
import org.apache.storm.jms.JmsProvider;
-import org.apache.storm.topology.base.BaseTickTupleAwareRichBolt;
-import org.apache.storm.utils.TupleUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseTickTupleAwareRichBolt;
import org.apache.storm.tuple.Tuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A JmsBolt receives <code>org.apache.storm.tuple.Tuple</code> objects from a Storm
@@ -124,9 +114,9 @@ public class JmsBolt extends BaseTickTupleAwareRichBolt {
*
* @param transactional
*/
-// public void setJmsTransactional(boolean transactional){
-// this.jmsTransactional = transactional;
-// }
+ // public void setJmsTransactional(boolean transactional){
+ // this.jmsTransactional = transactional;
+ // }
/**
* Sets whether or not tuples should be acknowledged by this
@@ -208,7 +198,7 @@ public class JmsBolt extends BaseTickTupleAwareRichBolt {
Destination dest = this.jmsProvider.destination();
this.connection = cf.createConnection();
this.session = connection.createSession(this.jmsTransactional,
- this.jmsAcknowledgeMode);
+ this.jmsAcknowledgeMode);
this.messageProducer = session.createProducer(dest);
connection.start();
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsMessageID.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsMessageID.java b/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsMessageID.java
index b78a41e..c069b7c 100644
--- a/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsMessageID.java
+++ b/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsMessageID.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jms.spout;
import java.io.Serializable;
@@ -25,19 +20,19 @@ public class JmsMessageID implements Comparable<JmsMessageID>, Serializable {
private Long sequence;
- public JmsMessageID(long sequence, String jmsID){
+ public JmsMessageID(long sequence, String jmsID) {
this.jmsID = jmsID;
this.sequence = sequence;
}
- public String getJmsID(){
+ public String getJmsID() {
return this.jmsID;
}
@Override
public int compareTo(JmsMessageID jmsMessageID) {
- return (int)(this.sequence - jmsMessageID.sequence);
+ return (int) (this.sequence - jmsMessageID.sequence);
}
@Override
@@ -47,8 +42,8 @@ public class JmsMessageID implements Comparable<JmsMessageID>, Serializable {
@Override
public boolean equals(Object o) {
- if(o instanceof JmsMessageID){
- JmsMessageID id = (JmsMessageID)o;
+ if (o instanceof JmsMessageID) {
+ JmsMessageID id = (JmsMessageID) o;
return this.jmsID.equals(id.jmsID);
} else {
return false;
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsSpout.java b/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsSpout.java
index 8973dbf..41d5636 100644
--- a/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsSpout.java
+++ b/external/storm-jms/src/main/java/org/apache/storm/jms/spout/JmsSpout.java
@@ -34,7 +34,6 @@ import javax.jms.Message;
import javax.jms.MessageConsumer;
import javax.jms.MessageListener;
import javax.jms.Session;
-
import org.apache.storm.Config;
import org.apache.storm.jms.JmsProvider;
import org.apache.storm.jms.JmsTupleProducer;
@@ -48,10 +47,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
/**
- * A Storm <code>Spout</code> implementation that listens to a JMS topic or
- * queue and outputs tuples based on the messages it receives.
+ * A Storm <code>Spout</code> implementation that listens to a JMS topic or queue and outputs tuples based on the messages it receives.
*
* <p><code>JmsSpout</code> instances rely on <code>JmsProducer</code>
* implementations to obtain the JMS
@@ -59,8 +56,7 @@ import org.slf4j.LoggerFactory;
* to connect to a JMS topic/queue.
*
* <p>When a <code>JmsSpout</code> receives a JMS message, it delegates to an
- * internal <code>JmsTupleProducer</code> instance to create a Storm tuple from
- * the incoming message.
+ * internal <code>JmsTupleProducer</code> instance to create a Storm tuple from the incoming message.
*
* <p>Typically, developers will supply a custom <code>JmsTupleProducer</code>
* implementation appropriate for the expected message content.
@@ -68,13 +64,19 @@ import org.slf4j.LoggerFactory;
@SuppressWarnings("serial")
public class JmsSpout extends BaseRichSpout implements MessageListener {
- /** The logger object instance for this class. */
+ /**
+ * The logger object instance for this class.
+ */
private static final Logger LOG = LoggerFactory.getLogger(JmsSpout.class);
- /** The logger of the recovery task. */
+ /**
+ * The logger of the recovery task.
+ */
private static final Logger RECOVERY_TASK_LOG = LoggerFactory.getLogger(RecoveryTask.class);
- /** Time to sleep between queue polling attempts. */
+ /**
+ * Time to sleep between queue polling attempts.
+ */
private static final int POLL_INTERVAL_MS = 50;
/**
@@ -82,88 +84,98 @@ public class JmsSpout extends BaseRichSpout implements MessageListener {
*/
private static final int DEFAULT_MESSAGE_TIMEOUT_SECS = 30;
- /** Time to wait before queuing the first recovery task. */
+ /**
+ * Time to wait before queuing the first recovery task.
+ */
private static final int RECOVERY_DELAY_MS = 10;
-
+ /**
+ * Used to safely recover failed JMS sessions across instances.
+ */
+ private final Serializable recoveryMutex = "RECOVERY_MUTEX";
/**
* The acknowledgment mode used for this instance.
*
* @see Session
*/
private int jmsAcknowledgeMode = Session.AUTO_ACKNOWLEDGE;
-
- /** Indicates whether or not this spout should run as a singleton. */
+ /**
+ * Indicates whether or not this spout should run as a singleton.
+ */
private boolean distributed = true;
-
- /** Used to generate tuples from incoming messages. */
+ /**
+ * Used to generate tuples from incoming messages.
+ */
private JmsTupleProducer tupleProducer;
-
- /** Encapsulates jms related classes needed to communicate with the mq. */
+ /**
+ * Encapsulates jms related classes needed to communicate with the mq.
+ */
private JmsProvider jmsProvider;
-
- /** Stores incoming messages for later sending. */
+ /**
+ * Stores incoming messages for later sending.
+ */
private LinkedBlockingQueue<Message> queue;
-
- /** Contains all message ids of messages that were not yet acked. */
+ /**
+ * Contains all message ids of messages that were not yet acked.
+ */
private TreeSet<JmsMessageID> toCommit;
-
- /** Maps between message ids of not-yet acked messages, and the messages. */
+ /**
+ * Maps between message ids of not-yet acked messages, and the messages.
+ */
private HashMap<JmsMessageID, Message> pendingMessages;
-
- /** Counter of handled messages. */
+ /**
+ * Counter of handled messages.
+ */
private long messageSequence = 0;
-
- /** The collector used to emit tuples. */
+ /**
+ * The collector used to emit tuples.
+ */
private SpoutOutputCollector collector;
-
- /** Connection to the jms queue. */
+ /**
+ * Connection to the jms queue.
+ */
private transient Connection connection;
-
- /** The active jms session. */
+ /**
+ * The active jms session.
+ */
private transient Session session;
-
- /** Indicates whether or not a message failed to be processed. */
+ /**
+ * Indicates whether or not a message failed to be processed.
+ */
private boolean hasFailures = false;
-
- /** Used to safely recover failed JMS sessions across instances. */
- private final Serializable recoveryMutex = "RECOVERY_MUTEX";
-
- /** Schedules recovery tasks periodically. */
+ /**
+ * Schedules recovery tasks periodically.
+ */
private Timer recoveryTimer = null;
- /** Time to wait between recovery attempts. */
+ /**
+ * Time to wait between recovery attempts.
+ */
private long recoveryPeriodMs = -1; // default to disabled
/**
- * Sets the JMS Session acknowledgement mode for the JMS session.
+ * Translate the {@code int} value of an acknowledgment to a {@code String}.
*
- * <p>Possible values:
- * <ul>
- * <li>javax.jms.Session.AUTO_ACKNOWLEDGE</li>
- * <li>javax.jms.Session.CLIENT_ACKNOWLEDGE</li>
- * <li>javax.jms.Session.DUPS_OK_ACKNOWLEDGE</li>
- * </ul>
+ * @param deliveryMode the mode to translate.
+ * @return its {@code String} explanation (name).
*
- * @param mode JMS Session Acknowledgement mode
- * @throws IllegalArgumentException if the mode is not recognized.
+ * @see Session
*/
- public void setJmsAcknowledgeMode(final int mode) {
- switch (mode) {
+ private static String toDeliveryModeString(int deliveryMode) {
+ switch (deliveryMode) {
case Session.AUTO_ACKNOWLEDGE:
+ return "AUTO_ACKNOWLEDGE";
case Session.CLIENT_ACKNOWLEDGE:
+ return "CLIENT_ACKNOWLEDGE";
case Session.DUPS_OK_ACKNOWLEDGE:
- break;
+ return "DUPS_OK_ACKNOWLEDGE";
default:
- throw new IllegalArgumentException(
- "Unknown Acknowledge mode: " + mode + " (See javax.jms.Session for valid values)");
+ return "UNKNOWN";
}
- this.jmsAcknowledgeMode = mode;
}
/**
- * Returns the JMS Session acknowledgement mode for the JMS session
- * associated with this spout. Can be either of:
+ * Returns the JMS Session acknowledgement mode for the JMS session associated with this spout. Can be either of:
* <ul>
* <li>{@link Session#AUTO_ACKNOWLEDGE}</li>
* <li>{@link Session#CLIENT_ACKNOWLEDGE}</li>
@@ -178,11 +190,37 @@ public class JmsSpout extends BaseRichSpout implements MessageListener {
}
/**
+ * Sets the JMS Session acknowledgement mode for the JMS session.
+ *
+ * <p>Possible values:
+ * <ul>
+ * <li>javax.jms.Session.AUTO_ACKNOWLEDGE</li>
+ * <li>javax.jms.Session.CLIENT_ACKNOWLEDGE</li>
+ * <li>javax.jms.Session.DUPS_OK_ACKNOWLEDGE</li>
+ * </ul>
+ *
+ * @param mode JMS Session Acknowledgement mode
+ * @throws IllegalArgumentException if the mode is not recognized.
+ */
+ public void setJmsAcknowledgeMode(final int mode) {
+ switch (mode) {
+ case Session.AUTO_ACKNOWLEDGE:
+ case Session.CLIENT_ACKNOWLEDGE:
+ case Session.DUPS_OK_ACKNOWLEDGE:
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unknown Acknowledge mode: " + mode + " (See javax.jms.Session for valid values)");
+
+ }
+ this.jmsAcknowledgeMode = mode;
+ }
+
+ /**
* Set {@link #jmsProvider}.
*
* <p>Set the <code>JmsProvider</code>
- * implementation that this Spout will use to connect to
- * a JMS <code>javax.jms.Desination</code>
+ * implementation that this Spout will use to connect to a JMS <code>javax.jms.Desination</code>
*
* @param provider the provider to use
*/
@@ -191,10 +229,8 @@ public class JmsSpout extends BaseRichSpout implements MessageListener {
}
/**
- * Set the <code>JmsTupleProducer</code>
- * implementation that will convert <code>javax.jms.Message</code>
- * object to <code>org.apache.storm.tuple.Values</code> objects
- * to be emitted.
+ * Set the <code>JmsTupleProducer</code> implementation that will convert <code>javax.jms.Message</code> object to
+ * <code>org.apache.storm.tuple.Values</code> objects to be emitted.
*
* @param producer the producer instance to use
*/
@@ -238,12 +274,12 @@ public class JmsSpout extends BaseRichSpout implements MessageListener {
}
// TODO get the default value from storm instead of hard coding 30 secs
Long topologyTimeout =
- ((Number) conf.getOrDefault(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, DEFAULT_MESSAGE_TIMEOUT_SECS)).longValue();
+ ((Number) conf.getOrDefault(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, DEFAULT_MESSAGE_TIMEOUT_SECS)).longValue();
if ((TimeUnit.SECONDS.toMillis(topologyTimeout)) > this.recoveryPeriodMs) {
LOG.warn("*** WARNING *** : "
- + "Recovery period (" + this.recoveryPeriodMs + " ms.) is less then the configured "
- + "'topology.message.timeout.secs' of " + topologyTimeout
- + " secs. This could lead to a message replay flood!");
+ + "Recovery period (" + this.recoveryPeriodMs + " ms.) is less then the configured "
+ + "'topology.message.timeout.secs' of " + topologyTimeout
+ + " secs. This could lead to a message replay flood!");
}
this.queue = new LinkedBlockingQueue<Message>();
this.toCommit = new TreeSet<JmsMessageID>();
@@ -288,10 +324,8 @@ public class JmsSpout extends BaseRichSpout implements MessageListener {
* Generate the next tuple from a message.
*
* <p>This method polls the queue that's being filled asynchronously by the
- * jms connection, every {@link #POLL_INTERVAL_MS} seconds. When a message
- * arrives, a {@link Values} (tuple) is generated using
- * {@link #tupleProducer}. It is emitted, and the message is saved to
- * {@link #toCommit} and {@link #pendingMessages} for later handling.
+ * jms connection, every {@link #POLL_INTERVAL_MS} seconds. When a message arrives, a {@link Values} (tuple) is generated using {@link
+ * #tupleProducer}. It is emitted, and the message is saved to {@link #toCommit} and {@link #pendingMessages} for later handling.
*/
public void nextTuple() {
Message msg = this.queue.poll();
@@ -332,8 +366,7 @@ public class JmsSpout extends BaseRichSpout implements MessageListener {
* Ack a successfully handled message by the matching {@link JmsMessageID}.
*
* <p>Acking means removing the message from the pending messages
- * collections, and if it was the oldest pending message -
- * ack it to the mq as well, so that it's the only one acked.
+ * collections, and if it was the oldest pending message - ack it to the mq as well, so that it's the only one acked.
*
* <p>Will only be called if we're transactional or not AUTO_ACKNOWLEDGE.
*/
@@ -392,8 +425,7 @@ public class JmsSpout extends BaseRichSpout implements MessageListener {
}
/**
- * Returns <code>true</code> if the spout has received failures
- * from which it has not yet recovered.
+ * Returns <code>true</code> if the spout has received failures from which it has not yet recovered.
*
* @return {@code true} if there were failures, {@code false} otherwise.
*/
@@ -409,8 +441,7 @@ public class JmsSpout extends BaseRichSpout implements MessageListener {
}
/**
- * Sets the periodicity of the timer task that
- * checks for failures and recovers the JMS session.
+ * Sets the periodicity of the timer task that checks for failures and recovers the JMS session.
*
* @param period desired wait period
*/
@@ -429,45 +460,21 @@ public class JmsSpout extends BaseRichSpout implements MessageListener {
* Sets the "distributed" mode of this spout.
*
* <p>If <code>true</code> multiple instances of this spout <i>may</i> be
- * created across the cluster
- * (depending on the "parallelism_hint" in the topology configuration).
+ * created across the cluster (depending on the "parallelism_hint" in the topology configuration).
*
* <p>Setting this value to <code>false</code> essentially means this spout
- * will run as a singleton within the cluster
- * ("parallelism_hint" will be ignored).
+ * will run as a singleton within the cluster ("parallelism_hint" will be ignored).
*
* <p>In general, this should be set to <code>false</code> if the underlying
* JMS destination is a topic, and <code>true</code> if it is a JMS queue.
*
- * @param isDistributed {@code true} if should be distributed, {@code false}
- * otherwise.
+ * @param isDistributed {@code true} if should be distributed, {@code false} otherwise.
*/
public void setDistributed(boolean isDistributed) {
this.distributed = isDistributed;
}
/**
- * Translate the {@code int} value of an acknowledgment to a {@code String}.
- *
- * @param deliveryMode the mode to translate.
- * @return its {@code String} explanation (name).
- * @see Session
- */
- private static String toDeliveryModeString(int deliveryMode) {
- switch (deliveryMode) {
- case Session.AUTO_ACKNOWLEDGE:
- return "AUTO_ACKNOWLEDGE";
- case Session.CLIENT_ACKNOWLEDGE:
- return "CLIENT_ACKNOWLEDGE";
- case Session.DUPS_OK_ACKNOWLEDGE:
- return "DUPS_OK_ACKNOWLEDGE";
- default:
- return "UNKNOWN";
-
- }
- }
-
- /**
* @return The currently active session.
*/
protected Session getSession() {
@@ -477,8 +484,7 @@ public class JmsSpout extends BaseRichSpout implements MessageListener {
/**
* Check if the subscription requires messages to be acked.
*
- * @return {@code true} if there is a pending messages state, {@code false}
- * otherwise.
+ * @return {@code true} if there is a pending messages state, {@code false} otherwise.
*/
private boolean isDurableSubscription() {
return (this.jmsAcknowledgeMode != Session.AUTO_ACKNOWLEDGE);
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsBatch.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsBatch.java b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsBatch.java
index c990058..5db4677 100644
--- a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsBatch.java
+++ b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsBatch.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jms.trident;
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsState.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsState.java b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsState.java
index bfb78b5..3611e78 100644
--- a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsState.java
+++ b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsState.java
@@ -1,35 +1,34 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jms.trident;
+import java.io.Serializable;
+import java.util.List;
+import javax.jms.Connection;
+import javax.jms.ConnectionFactory;
+import javax.jms.Destination;
+import javax.jms.JMSException;
+import javax.jms.Message;
+import javax.jms.MessageProducer;
+import javax.jms.Session;
import org.apache.storm.jms.JmsMessageProducer;
import org.apache.storm.jms.JmsProvider;
import org.apache.storm.topology.FailedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.tuple.TridentTuple;
-
-import javax.jms.*;
-import java.io.Serializable;
-import java.lang.IllegalStateException;
-import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class JmsState implements State {
@@ -44,35 +43,8 @@ public class JmsState implements State {
this.options = options;
}
- public static class Options implements Serializable {
- private JmsProvider jmsProvider;
- private JmsMessageProducer msgProducer;
- private int jmsAcknowledgeMode = Session.AUTO_ACKNOWLEDGE;
- private boolean jmsTransactional = true;
-
- public Options withJmsProvider(JmsProvider provider) {
- this.jmsProvider = provider;
- return this;
- }
-
- public Options withMessageProducer(JmsMessageProducer msgProducer) {
- this.msgProducer = msgProducer;
- return this;
- }
-
- public Options withJmsAcknowledgeMode(int jmsAcknowledgeMode) {
- this.jmsAcknowledgeMode = jmsAcknowledgeMode;
- return this;
- }
-
- public Options withJmsTransactional(boolean jmsTransactional) {
- this.jmsTransactional = jmsTransactional;
- return this;
- }
- }
-
protected void prepare() {
- if(this.options.jmsProvider == null || this.options.msgProducer == null){
+ if (this.options.jmsProvider == null || this.options.msgProducer == null) {
throw new IllegalStateException("JMS Provider and MessageProducer not set.");
}
LOG.debug("Connecting JMS..");
@@ -81,7 +53,7 @@ public class JmsState implements State {
Destination dest = this.options.jmsProvider.destination();
this.connection = cf.createConnection();
this.session = connection.createSession(this.options.jmsTransactional,
- this.options.jmsAcknowledgeMode);
+ this.options.jmsAcknowledgeMode);
this.messageProducer = session.createProducer(dest);
connection.start();
@@ -97,10 +69,10 @@ public class JmsState implements State {
@Override
public void commit(Long aLong) {
LOG.debug("Committing JMS transaction.");
- if(this.options.jmsTransactional) {
+ if (this.options.jmsTransactional) {
try {
session.commit();
- } catch(JMSException e){
+ } catch (JMSException e) {
LOG.error("JMS Session commit failed.", e);
}
}
@@ -108,7 +80,7 @@ public class JmsState implements State {
public void updateState(List<TridentTuple> tuples, TridentCollector collector) throws JMSException {
try {
- for(TridentTuple tuple : tuples) {
+ for (TridentTuple tuple : tuples) {
Message msg = this.options.msgProducer.toMessage(this.session, tuple);
if (msg != null) {
if (msg.getJMSDestination() != null) {
@@ -120,10 +92,37 @@ public class JmsState implements State {
}
} catch (JMSException e) {
LOG.warn("Failed to send jmd message for a trident batch ", e);
- if(this.options.jmsTransactional) {
+ if (this.options.jmsTransactional) {
session.rollback();
}
throw new FailedException("Failed to write tuples", e);
}
}
+
+ public static class Options implements Serializable {
+ private JmsProvider jmsProvider;
+ private JmsMessageProducer msgProducer;
+ private int jmsAcknowledgeMode = Session.AUTO_ACKNOWLEDGE;
+ private boolean jmsTransactional = true;
+
+ public Options withJmsProvider(JmsProvider provider) {
+ this.jmsProvider = provider;
+ return this;
+ }
+
+ public Options withMessageProducer(JmsMessageProducer msgProducer) {
+ this.msgProducer = msgProducer;
+ return this;
+ }
+
+ public Options withJmsAcknowledgeMode(int jmsAcknowledgeMode) {
+ this.jmsAcknowledgeMode = jmsAcknowledgeMode;
+ return this;
+ }
+
+ public Options withJmsTransactional(boolean jmsTransactional) {
+ this.jmsTransactional = jmsTransactional;
+ return this;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsStateFactory.java b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsStateFactory.java
index 4123752..b72f88c 100644
--- a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsStateFactory.java
+++ b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsStateFactory.java
@@ -1,28 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jms.trident;
+import java.util.Map;
import org.apache.storm.task.IMetricsContext;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.state.StateFactory;
-import java.util.Map;
-
public class JmsStateFactory implements StateFactory {
private JmsState.Options options;
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsUpdater.java b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsUpdater.java
index a2709a4..92b8a95 100644
--- a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsUpdater.java
+++ b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/JmsUpdater.java
@@ -1,31 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jms.trident;
+import java.util.List;
+import javax.jms.JMSException;
import org.apache.storm.topology.FailedException;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseStateUpdater;
import org.apache.storm.trident.tuple.TridentTuple;
-import javax.jms.JMSException;
-import java.util.List;
-
-public class JmsUpdater extends BaseStateUpdater<JmsState> {
+public class JmsUpdater extends BaseStateUpdater<JmsState> {
@Override
public void updateState(JmsState jmsState, List<TridentTuple> tuples, TridentCollector collector) {
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/TridentJmsSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/TridentJmsSpout.java b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/TridentJmsSpout.java
index afdc0b2..de7c182 100644
--- a/external/storm-jms/src/main/java/org/apache/storm/jms/trident/TridentJmsSpout.java
+++ b/external/storm-jms/src/main/java/org/apache/storm/jms/trident/TridentJmsSpout.java
@@ -1,27 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jms.trident;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
-
import javax.jms.Connection;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
@@ -30,23 +24,21 @@ import javax.jms.Message;
import javax.jms.MessageConsumer;
import javax.jms.MessageListener;
import javax.jms.Session;
-
+import org.apache.storm.Config;
+import org.apache.storm.generated.StreamInfo;
import org.apache.storm.jms.JmsProvider;
import org.apache.storm.jms.JmsTupleProducer;
-import org.apache.storm.utils.Utils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsGetter;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.spout.ITridentSpout;
import org.apache.storm.trident.topology.TransactionAttempt;
-import org.apache.storm.Config;
-import org.apache.storm.generated.StreamInfo;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.OutputFieldsGetter;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.RotatingMap;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Trident implementation of the JmsSpout
@@ -56,29 +48,52 @@ import org.apache.storm.utils.RotatingMap;
public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
public static final String MAX_BATCH_SIZE_CONF = "topology.spout.max.batch.size";
-
+
public static final int DEFAULT_BATCH_SIZE = 1000;
private static final long serialVersionUID = -3469351154693356655L;
-
+ private static int nameIndex = 1;
private JmsTupleProducer tupleProducer;
-
private JmsProvider jmsProvider;
-
private int jmsAcknowledgeMode;
-
private String name;
- private static int nameIndex = 1;
-
/**
* Create a TridentJmsSpout with a default name and acknowledge mode of AUTO_ACKNOWLEDGE
*/
public TridentJmsSpout() {
- this.name = "JmsSpout_"+(nameIndex++);
+ this.name = "JmsSpout_" + (nameIndex++);
this.jmsAcknowledgeMode = Session.AUTO_ACKNOWLEDGE;
}
-
+
+ /**
+ * Return a friendly string for the given JMS acknowledge mode, or throw an IllegalArgumentException if
+ * the mode is not recognized.
+ * <p/>
+ * Possible values:
+ * <ul>
+ * <li>javax.jms.Session.AUTO_ACKNOWLEDGE</li>
+ * <li>javax.jms.Session.CLIENT_ACKNOWLEDGE</li>
+ * <li>javax.jms.Session.DUPS_OK_ACKNOWLEDGE</li>
+ * </ul>
+ * @param acknowledgeMode A valid JMS acknowledge mode
+ * @return A friendly string describing the acknowledge mode
+ * @throws IllegalArgumentException if the mode is not recognized
+ */
+ private static final String toDeliveryModeString(int acknowledgeMode) {
+ switch (acknowledgeMode) {
+ case Session.AUTO_ACKNOWLEDGE:
+ return "AUTO_ACKNOWLEDGE";
+ case Session.CLIENT_ACKNOWLEDGE:
+ return "CLIENT_ACKNOWLEDGE";
+ case Session.DUPS_OK_ACKNOWLEDGE:
+ return "DUPS_OK_ACKNOWLEDGE";
+ default:
+ throw new IllegalArgumentException(
+ "Unknown JMS Acknowledge mode " + acknowledgeMode + " (See javax.jms.Session for valid values)");
+ }
+ }
+
/**
* Set the name for this spout, to improve log identification
* @param name The name to be used in log messages
@@ -88,25 +103,25 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
this.name = name;
return this;
}
-
+
/**
* Set the <code>JmsProvider</code>
- * implementation that this Spout will use to connect to
+ * implementation that this Spout will use to connect to
* a JMS <code>javax.jms.Desination</code>
- *
+ *
* @param provider
*/
- public TridentJmsSpout withJmsProvider(JmsProvider provider){
+ public TridentJmsSpout withJmsProvider(JmsProvider provider) {
this.jmsProvider = provider;
return this;
}
-
+
/**
* Set the <code>JmsTupleProducer</code>
* implementation that will convert <code>javax.jms.Message</code>
* object to <code>backtype.storm.tuple.Values</code> objects
* to be emitted.
- *
+ *
* @param tupleProducer
* @return This spout
*/
@@ -114,7 +129,7 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
this.tupleProducer = tupleProducer;
return this;
}
-
+
/**
* Set the JMS acknowledge mode for messages being processed by this spout.
* <p/>
@@ -133,37 +148,10 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
this.jmsAcknowledgeMode = jmsAcknowledgeMode;
return this;
}
-
- /**
- * Return a friendly string for the given JMS acknowledge mode, or throw an IllegalArgumentException if
- * the mode is not recognized.
- * <p/>
- * Possible values:
- * <ul>
- * <li>javax.jms.Session.AUTO_ACKNOWLEDGE</li>
- * <li>javax.jms.Session.CLIENT_ACKNOWLEDGE</li>
- * <li>javax.jms.Session.DUPS_OK_ACKNOWLEDGE</li>
- * </ul>
- * @param acknowledgeMode A valid JMS acknowledge mode
- * @return A friendly string describing the acknowledge mode
- * @throws IllegalArgumentException if the mode is not recognized
- */
- private static final String toDeliveryModeString(int acknowledgeMode) {
- switch (acknowledgeMode) {
- case Session.AUTO_ACKNOWLEDGE:
- return "AUTO_ACKNOWLEDGE";
- case Session.CLIENT_ACKNOWLEDGE:
- return "CLIENT_ACKNOWLEDGE";
- case Session.DUPS_OK_ACKNOWLEDGE:
- return "DUPS_OK_ACKNOWLEDGE";
- default:
- throw new IllegalArgumentException("Unknown JMS Acknowledge mode " + acknowledgeMode + " (See javax.jms.Session for valid values)");
- }
- }
-
+
@Override
public ITridentSpout.BatchCoordinator<JmsBatch> getCoordinator(
- String txStateId, Map<String, Object> conf, TopologyContext context) {
+ String txStateId, Map<String, Object> conf, TopologyContext context) {
return new JmsBatchCoordinator(name);
}
@@ -185,10 +173,10 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
if (streamInfo == null) {
throw new IllegalArgumentException("Jms Tuple producer has not declared output fields for the default stream");
}
-
+
return new Fields(streamInfo.get_output_fields());
}
-
+
/**
* The JmsEmitter class listens for incoming messages and stores them in a blocking queue. On each invocation of emit,
* the queued messages are emitted as a batch.
@@ -201,16 +189,15 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
private final Session session;
private final RotatingMap<Long, List<Message>> batchMessageMap; // Maps transaction Ids to JMS message ids.
-
+
private final long rotateTimeMillis;
private final int maxBatchSize;
private final String name;
-
- private long lastRotate;
-
private final Logger LOG = LoggerFactory.getLogger(JmsEmitter.class);
-
- public JmsEmitter(String name, JmsProvider jmsProvider, JmsTupleProducer tupleProducer, int jmsAcknowledgeMode, Map<String, Object> conf) {
+ private long lastRotate;
+
+ public JmsEmitter(String name, JmsProvider jmsProvider, JmsTupleProducer tupleProducer, int jmsAcknowledgeMode,
+ Map<String, Object> conf) {
if (jmsProvider == null) {
throw new IllegalStateException("JMS provider has not been set.");
}
@@ -220,11 +207,11 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
this.queue = new LinkedBlockingQueue<Message>();
this.name = name;
-
+
batchMessageMap = new RotatingMap<Long, List<Message>>(3);
- rotateTimeMillis = 1000L * ((Number)conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue();
+ rotateTimeMillis = 1000L * ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue();
lastRotate = System.currentTimeMillis();
-
+
Number batchSize = (Number) conf.get(MAX_BATCH_SIZE_CONF);
maxBatchSize = batchSize != null ? batchSize.intValue() : DEFAULT_BATCH_SIZE;
@@ -237,40 +224,41 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
consumer.setMessageListener(this);
this.connection.start();
- LOG.info("Created JmsEmitter with max batch size "+maxBatchSize+" rotate time "+rotateTimeMillis+"ms and destination "+dest+" for "+name);
+ LOG.info(
+ "Created JmsEmitter with max batch size " + maxBatchSize + " rotate time " + rotateTimeMillis + "ms and destination " +
+ dest + " for " + name);
} catch (Exception e) {
LOG.warn("Error creating JMS connection.", e);
throw new IllegalStateException("Could not create JMS connection for spout ", e);
}
-
+
}
-
+
@Override
public void success(TransactionAttempt tx) {
-
+
@SuppressWarnings("unchecked")
List<Message> messages = (List<Message>) batchMessageMap.remove(tx.getTransactionId());
-
+
if (messages != null) {
if (!messages.isEmpty()) {
- LOG.debug("Success for batch with transaction id "+tx.getTransactionId()+"/"+tx.getAttemptId()+" for "+name);
+ LOG.debug("Success for batch with transaction id " + tx.getTransactionId() + "/" + tx.getAttemptId() + " for " + name);
}
-
- for (Message msg: messages) {
+
+ for (Message msg : messages) {
String messageId = "UnknownId";
-
+
try {
messageId = msg.getJMSMessageID();
msg.acknowledge();
- LOG.trace("Acknowledged message "+messageId);
+ LOG.trace("Acknowledged message " + messageId);
} catch (JMSException e) {
- LOG.warn("Failed to acknowledge message "+messageId, e);
+ LOG.warn("Failed to acknowledge message " + messageId, e);
}
}
- }
- else {
- LOG.warn("No messages found in batch with transaction id "+tx.getTransactionId()+"/"+tx.getAttemptId());
+ } else {
+ LOG.warn("No messages found in batch with transaction id " + tx.getTransactionId() + "/" + tx.getAttemptId());
}
}
@@ -282,19 +270,18 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
* @param messages The list of messages to fail.
*/
private void fail(Long transactionId, List<Message> messages) {
- LOG.debug("Failure for batch with transaction id "+transactionId+" for "+name);
+ LOG.debug("Failure for batch with transaction id " + transactionId + " for " + name);
if (messages != null) {
- for (Message msg: messages) {
+ for (Message msg : messages) {
try {
- LOG.trace("Failed message "+msg.getJMSMessageID());
+ LOG.trace("Failed message " + msg.getJMSMessageID());
} catch (JMSException e) {
LOG.warn("Could not identify failed message ", e);
}
}
+ } else {
+ LOG.warn("Failed batch has no messages with transaction id " + transactionId);
}
- else {
- LOG.warn("Failed batch has no messages with transaction id "+transactionId);
- }
}
@Override
@@ -305,37 +292,37 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
this.connection.close();
} catch (JMSException e) {
LOG.warn("Error closing JMS connection.", e);
- }
+ }
}
@Override
public void emitBatch(TransactionAttempt tx, JmsBatch coordinatorMeta,
- TridentCollector collector) {
-
+ TridentCollector collector) {
+
long now = System.currentTimeMillis();
- if(now - lastRotate > rotateTimeMillis) {
+ if (now - lastRotate > rotateTimeMillis) {
Map<Long, List<Message>> failed = batchMessageMap.rotate();
- for(Long id: failed.keySet()) {
- LOG.warn("TIMED OUT batch with transaction id "+id+" for "+name);
+ for (Long id : failed.keySet()) {
+ LOG.warn("TIMED OUT batch with transaction id " + id + " for " + name);
fail(id, failed.get(id));
}
lastRotate = now;
}
-
- if(batchMessageMap.containsKey(tx.getTransactionId())) {
- LOG.warn("FAILED duplicate batch with transaction id "+tx.getTransactionId()+"/"+tx.getAttemptId()+" for "+name);
+
+ if (batchMessageMap.containsKey(tx.getTransactionId())) {
+ LOG.warn("FAILED duplicate batch with transaction id " + tx.getTransactionId() + "/" + tx.getAttemptId() + " for " + name);
fail(tx.getTransactionId(), batchMessageMap.get(tx.getTransactionId()));
}
-
+
List<Message> batchMessages = new ArrayList<Message>();
-
- for (int index=0; index<maxBatchSize; index++) {
+
+ for (int index = 0; index < maxBatchSize; index++) {
Message msg = queue.poll();
if (msg == null) {
Utils.sleep(50); // Back off
break;
}
-
+
try {
if (TridentJmsSpout.this.jmsAcknowledgeMode != Session.AUTO_ACKNOWLEDGE) {
batchMessages.add(msg);
@@ -343,15 +330,17 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
Values tuple = tupleProducer.toTuple(msg);
collector.emit(tuple);
} catch (JMSException e) {
- LOG.warn("Failed to emit message, could not retrieve data for "+name+": "+e );
+ LOG.warn("Failed to emit message, could not retrieve data for " + name + ": " + e);
}
}
-
+
if (!batchMessages.isEmpty()) {
- LOG.debug("Emitting batch with transaction id "+tx.getTransactionId()+"/"+tx.getAttemptId()+" and size "+batchMessages.size()+" for "+name);
- }
- else {
- LOG.trace("No items to acknowledge for batch with transaction id "+tx.getTransactionId()+"/"+tx.getAttemptId()+" for "+name);
+ LOG.debug("Emitting batch with transaction id " + tx.getTransactionId() + "/" + tx.getAttemptId() + " and size " +
+ batchMessages.size() + " for " + name);
+ } else {
+ LOG.trace(
+ "No items to acknowledge for batch with transaction id " + tx.getTransactionId() + "/" + tx.getAttemptId() + " for " +
+ name);
}
batchMessageMap.put(tx.getTransactionId(), batchMessages);
}
@@ -365,9 +354,9 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
}
this.queue.offer(msg);
}
-
+
}
-
+
/**
* Bare implementation of a BatchCoordinator, returning a null JmsBatch object
*
@@ -375,17 +364,17 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
private class JmsBatchCoordinator implements BatchCoordinator<JmsBatch> {
private final String name;
-
+
private final Logger LOG = LoggerFactory.getLogger(JmsBatchCoordinator.class);
public JmsBatchCoordinator(String name) {
this.name = name;
- LOG.info("Created batch coordinator for "+name);
+ LOG.info("Created batch coordinator for " + name);
}
-
+
@Override
public JmsBatch initializeTransaction(long txid, JmsBatch prevMetadata, JmsBatch curMetadata) {
- LOG.debug("Initialise transaction "+txid+" for "+name);
+ LOG.debug("Initialise transaction " + txid + " for " + name);
return null;
}
@@ -401,7 +390,7 @@ public class TridentJmsSpout implements ITridentSpout<JmsBatch> {
@Override
public void close() {
}
-
+
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java
index b6406c8..9f967f8 100644
--- a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java
+++ b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java
@@ -15,16 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.jms.spout;
-import org.apache.storm.Config;
-import org.apache.storm.jms.JmsProvider;
-import org.apache.storm.spout.SpoutOutputCollector;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+package org.apache.storm.jms.spout;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.util.HashMap;
+import java.util.Map;
import javax.jms.ConnectionFactory;
import javax.jms.Destination;
import javax.jms.JMSException;
@@ -32,15 +30,17 @@ import javax.jms.Message;
import javax.jms.MessageProducer;
import javax.jms.Session;
import javax.jms.TextMessage;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.util.HashMap;
-import java.util.Map;
+import org.apache.storm.Config;
+import org.apache.storm.jms.JmsProvider;
+import org.apache.storm.spout.SpoutOutputCollector;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class JmsSpoutTest {
private static final Logger LOG =
- LoggerFactory.getLogger(JmsSpoutTest.class);
+ LoggerFactory.getLogger(JmsSpoutTest.class);
@Test
public void testFailure() throws JMSException, Exception {
@@ -85,9 +85,8 @@ public class JmsSpoutTest {
}
/**
- * Make sure that {@link JmsSpout#open} returns correctly regardless of
- * the type of {@link Number} that is the value of
- * {@link Config#TOPOLOGY_MESSAGE_TIMEOUT_SECS}.
+ * Make sure that {@link JmsSpout#open} returns correctly regardless of the type of {@link Number} that is the value of {@link
+ * Config#TOPOLOGY_MESSAGE_TIMEOUT_SECS}.
*/
@Test
public void testOpenWorksMultipleTypesOfNumberObjects() throws Exception {
@@ -118,8 +117,8 @@ public class JmsSpoutTest {
Destination destination) throws JMSException {
Session mySess = connectionFactory.createConnection().createSession(
- false,
- Session.CLIENT_ACKNOWLEDGE);
+ false,
+ Session.CLIENT_ACKNOWLEDGE);
MessageProducer producer = mySess.createProducer(destination);
TextMessage msg = mySess.createTextMessage();
msg.setText("Hello World");
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockJmsProvider.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockJmsProvider.java b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockJmsProvider.java
index 3ba0853..ca9733d 100644
--- a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockJmsProvider.java
+++ b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockJmsProvider.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.jms.spout;
import javax.jms.ConnectionFactory;
@@ -22,9 +23,7 @@ import javax.jms.Destination;
import javax.naming.Context;
import javax.naming.InitialContext;
import javax.naming.NamingException;
-
import org.apache.activemq.ActiveMQConnectionFactory;
-
import org.apache.storm.jms.JmsProvider;
public class MockJmsProvider implements JmsProvider {
@@ -32,30 +31,34 @@ public class MockJmsProvider implements JmsProvider {
private ConnectionFactory connectionFactory = null;
private Destination destination = null;
-
- public MockJmsProvider() throws NamingException{
- this.connectionFactory = new ActiveMQConnectionFactory("vm://localhost?broker.persistent=false");
+
+ public MockJmsProvider() throws NamingException {
+ this.connectionFactory = new ActiveMQConnectionFactory("vm://localhost?broker.persistent=false");
Context jndiContext = new InitialContext();
- this.destination = (Destination) jndiContext.lookup("dynamicQueues/FOO.BAR");
+ this.destination = (Destination) jndiContext.lookup("dynamicQueues/FOO.BAR");
}
-
+
/**
* Provides the JMS <code>ConnectionFactory</code>
+ *
* @return the connection factory
+ *
* @throws Exception
*/
- public ConnectionFactory connectionFactory() throws Exception{
+ public ConnectionFactory connectionFactory() throws Exception {
return this.connectionFactory;
}
/**
* Provides the <code>Destination</code> (topic or queue) from which the
* <code>JmsSpout</code> will receive messages.
+ *
* @return
+ *
* @throws Exception
*/
- public Destination destination() throws Exception{
+ public Destination destination() throws Exception {
return this.destination;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockSpoutOutputCollector.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockSpoutOutputCollector.java b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockSpoutOutputCollector.java
index 4e05646..60710bc 100644
--- a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockSpoutOutputCollector.java
+++ b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockSpoutOutputCollector.java
@@ -15,11 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.jms.spout;
import java.util.ArrayList;
import java.util.List;
-
import org.apache.storm.spout.ISpoutOutputCollector;
public class MockSpoutOutputCollector implements ISpoutOutputCollector {
@@ -45,11 +45,11 @@ public class MockSpoutOutputCollector implements ISpoutOutputCollector {
public void reportError(Throwable error) {
}
- public boolean emitted(){
+ public boolean emitted() {
return this.emitted;
}
- public void reset(){
+ public void reset() {
this.emitted = false;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/95602b1b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockTupleProducer.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockTupleProducer.java b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockTupleProducer.java
index ea571fc..70e04ff 100644
--- a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockTupleProducer.java
+++ b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/MockTupleProducer.java
@@ -15,12 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.jms.spout;
import javax.jms.JMSException;
import javax.jms.Message;
import javax.jms.TextMessage;
-
import org.apache.storm.jms.JmsTupleProducer;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
[27/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-hdfs
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileRotationPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileRotationPolicy.java
index c69f312..ad9d7aa 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileRotationPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileRotationPolicy.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.trident.rotation;
-import org.apache.storm.trident.tuple.TridentTuple;
+package org.apache.storm.hdfs.trident.rotation;
import java.io.Serializable;
+import org.apache.storm.trident.tuple.TridentTuple;
/**
* Used by the HdfsBolt to decide when to rotate files.
@@ -44,7 +38,7 @@ public interface FileRotationPolicy extends Serializable {
/**
* Check if a file rotation should be performed based on
* the offset at which file is being written.
- *
+ *
* @param offset the current offset of file being written
* @return true if a file rotation should be performed.
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileSizeRotationPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileSizeRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileSizeRotationPolicy.java
index 70865b2..2a512c4 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileSizeRotationPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/FileSizeRotationPolicy.java
@@ -1,26 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.trident.rotation;
+import org.apache.storm.trident.tuple.TridentTuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.storm.trident.tuple.TridentTuple;
/**
* File rotation policy that will rotate files when a certain
@@ -36,32 +31,11 @@ import org.apache.storm.trident.tuple.TridentTuple;
*/
public class FileSizeRotationPolicy implements FileRotationPolicy {
private static final Logger LOG = LoggerFactory.getLogger(FileSizeRotationPolicy.class);
-
- public static enum Units {
-
- KB((long)Math.pow(2, 10)),
- MB((long)Math.pow(2, 20)),
- GB((long)Math.pow(2, 30)),
- TB((long)Math.pow(2, 40));
-
- private long byteCount;
-
- private Units(long byteCount){
- this.byteCount = byteCount;
- }
-
- public long getByteCount(){
- return byteCount;
- }
- }
-
private long maxBytes;
-
private long lastOffset = 0;
private long currentBytesWritten = 0;
-
- public FileSizeRotationPolicy(float count, Units units){
- this.maxBytes = (long)(count * units.getByteCount());
+ public FileSizeRotationPolicy(float count, Units units) {
+ this.maxBytes = (long) (count * units.getByteCount());
}
@Override
@@ -91,4 +65,22 @@ public class FileSizeRotationPolicy implements FileRotationPolicy {
public long getMaxBytes() {
return maxBytes;
}
+
+ public static enum Units {
+
+ KB((long) Math.pow(2, 10)),
+ MB((long) Math.pow(2, 20)),
+ GB((long) Math.pow(2, 30)),
+ TB((long) Math.pow(2, 40));
+
+ private long byteCount;
+
+ private Units(long byteCount) {
+ this.byteCount = byteCount;
+ }
+
+ public long getByteCount() {
+ return byteCount;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/NoRotationPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/NoRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/NoRotationPolicy.java
index 4c65acd..f6fedb9 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/NoRotationPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/NoRotationPolicy.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.trident.rotation;
import org.apache.storm.trident.tuple.TridentTuple;
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java
index 7278c9a..2508a07 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java
@@ -15,43 +15,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.hdfs.trident.rotation;
-import org.apache.storm.trident.tuple.TridentTuple;
+package org.apache.storm.hdfs.trident.rotation;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.storm.trident.tuple.TridentTuple;
public class TimedRotationPolicy implements FileRotationPolicy {
- public static enum TimeUnit {
-
- SECONDS((long)1000),
- MINUTES((long)1000*60),
- HOURS((long)1000*60*60),
- DAYS((long)1000*60*60*24);
-
- private long milliSeconds;
-
- private TimeUnit(long milliSeconds){
- this.milliSeconds = milliSeconds;
- }
-
- public long getMilliSeconds(){
- return milliSeconds;
- }
- }
-
private long interval;
private Timer rotationTimer;
private AtomicBoolean rotationTimerTriggered = new AtomicBoolean();
-
-
- public TimedRotationPolicy(float count, TimeUnit units){
- this.interval = (long)(count * units.getMilliSeconds());
+ public TimedRotationPolicy(float count, TimeUnit units) {
+ this.interval = (long) (count * units.getMilliSeconds());
}
+
/**
* Called for every tuple the HdfsBolt executes.
*
@@ -77,7 +58,7 @@ public class TimedRotationPolicy implements FileRotationPolicy {
rotationTimerTriggered.set(false);
}
- public long getInterval(){
+ public long getInterval() {
return this.interval;
}
@@ -95,4 +76,22 @@ public class TimedRotationPolicy implements FileRotationPolicy {
};
rotationTimer.scheduleAtFixedRate(task, interval, interval);
}
+
+ public static enum TimeUnit {
+
+ SECONDS((long) 1000),
+ MINUTES((long) 1000 * 60),
+ HOURS((long) 1000 * 60 * 60),
+ DAYS((long) 1000 * 60 * 60 * 24);
+
+ private long milliSeconds;
+
+ private TimeUnit(long milliSeconds) {
+ this.milliSeconds = milliSeconds;
+ }
+
+ public long getMilliSeconds() {
+ return milliSeconds;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/CountSyncPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/CountSyncPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/CountSyncPolicy.java
index 3b4e32e..15f0c6b 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/CountSyncPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/CountSyncPolicy.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.trident.sync;
@@ -29,7 +24,7 @@ public class CountSyncPolicy implements SyncPolicy {
private int count;
private int executeCount = 0;
- public CountSyncPolicy(int count){
+ public CountSyncPolicy(int count) {
this.count = count;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/SyncPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/SyncPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/SyncPolicy.java
index d87da9c..ba39724 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/SyncPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/sync/SyncPolicy.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.trident.sync;
-import org.apache.storm.trident.tuple.TridentTuple;
+package org.apache.storm.hdfs.trident.sync;
import java.io.Serializable;
+import org.apache.storm.trident.tuple.TridentTuple;
/**
* Interface for controlling when the HdfsBolt
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java
index b588f5b..5cc7753 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java
@@ -1,43 +1,41 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.avro;
import org.apache.avro.Schema;
import org.junit.Assert;
-import org.junit.Test;
-
import org.junit.BeforeClass;
+import org.junit.Test;
public class TestFixedAvroSerializer {
//These should match FixedAvroSerializer.config in the test resources
private static final String schemaString1 = "{\"type\":\"record\"," +
- "\"name\":\"stormtest1\"," +
- "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"}," +
- "{ \"name\":\"int1\", \"type\":\"int\" }]}";
+ "\"name\":\"stormtest1\"," +
+ "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"}," +
+ "{ \"name\":\"int1\", \"type\":\"int\" }]}";
private static final String schemaString2 = "{\"type\":\"record\"," +
- "\"name\":\"stormtest2\"," +
- "\"fields\":[{\"name\":\"foobar1\",\"type\":\"string\"}," +
- "{ \"name\":\"intint1\", \"type\":\"int\" }]}";
+ "\"name\":\"stormtest2\"," +
+ "\"fields\":[{\"name\":\"foobar1\",\"type\":\"string\"}," +
+ "{ \"name\":\"intint1\", \"type\":\"int\" }]}";
private static Schema schema1;
private static Schema schema2;
final AvroSchemaRegistry reg;
+ public TestFixedAvroSerializer() throws Exception {
+ reg = new FixedAvroSerializer();
+ }
+
@BeforeClass
public static void setupClass() {
@@ -48,17 +46,13 @@ public class TestFixedAvroSerializer {
schema2 = parser.parse(schemaString2);
}
- public TestFixedAvroSerializer() throws Exception{
- reg = new FixedAvroSerializer();
- }
-
@Test
public void testSchemas() {
testTheSchema(schema1);
testTheSchema(schema2);
}
- @Test
+ @Test
public void testDifferentFPs() {
String fp1 = reg.getFingerprint(schema1);
String fp2 = reg.getFingerprint(schema2);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java
index fb97782..cf02dcb 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.avro;
import org.apache.avro.Schema;
@@ -24,13 +19,13 @@ import org.junit.Test;
public class TestGenericAvroSerializer {
private static final String schemaString1 = "{\"type\":\"record\"," +
- "\"name\":\"stormtest1\"," +
- "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"}," +
- "{ \"name\":\"int1\", \"type\":\"int\" }]}";
+ "\"name\":\"stormtest1\"," +
+ "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"}," +
+ "{ \"name\":\"int1\", \"type\":\"int\" }]}";
private static final String schemaString2 = "{\"type\":\"record\"," +
- "\"name\":\"stormtest2\"," +
- "\"fields\":[{\"name\":\"foobar1\",\"type\":\"string\"}," +
- "{ \"name\":\"intint1\", \"type\":\"int\" }]}";
+ "\"name\":\"stormtest2\"," +
+ "\"fields\":[{\"name\":\"foobar1\",\"type\":\"string\"}," +
+ "{ \"name\":\"intint1\", \"type\":\"int\" }]}";
private static Schema schema1;
private static Schema schema2;
@@ -52,7 +47,7 @@ public class TestGenericAvroSerializer {
testTheSchema(schema2);
}
- @Test
+ @Test
public void testDifferentFPs() {
String fp1 = reg.getFingerprint(schema1);
String fp2 = reg.getFingerprint(schema2);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java
index cac55c9..a919c92 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java
@@ -1,72 +1,77 @@
-
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt;
-import org.apache.storm.Config;
-import org.apache.storm.task.GeneralTopologyContext;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.TopologyBuilder;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.tuple.TupleImpl;
-import org.apache.storm.tuple.Values;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericRecordBuilder;
+import org.apache.avro.file.DataFileReader;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.GenericRecordBuilder;
import org.apache.avro.io.DatumReader;
-import org.apache.avro.file.DataFileReader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.storm.Config;
import org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
-import org.junit.Before;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.apache.storm.task.GeneralTopologyContext;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.TopologyBuilder;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.TupleImpl;
+import org.apache.storm.tuple.Values;
import org.junit.After;
-import org.junit.Test;
import org.junit.Assert;
-
-import org.mockito.Mock;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FSDataInputStream;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.HashMap;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
+import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class AvroGenericRecordBoltTest {
+ private static final String testRoot = "/unittest";
+ private static final String schemaV1 = "{\"type\":\"record\","
+ + "\"name\":\"myrecord\","
+ + "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"},"
+ + "{ \"name\":\"int1\", \"type\":\"int\" }]}";
+ private static final String schemaV2 = "{\"type\":\"record\","
+ + "\"name\":\"myrecord\","
+ + "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"},"
+ + "{ \"name\":\"bar\", \"type\":\"string\", \"default\":\"baz\" },"
+ + "{ \"name\":\"int1\", \"type\":\"int\" }]}";
+ private static Schema schema1;
+ private static Schema schema2;
+ private static Tuple tuple1;
+ private static Tuple tuple2;
@Rule
public MiniDFSClusterRule dfsClusterRule = new MiniDFSClusterRule(() -> {
Configuration conf = new Configuration();
@@ -81,24 +86,8 @@ public class AvroGenericRecordBoltTest {
private OutputCollector collector;
@Mock
private TopologyContext topologyContext;
-
private DistributedFileSystem fs;
private String hdfsURI;
- private static final String testRoot = "/unittest";
- private static Schema schema1;
- private static Schema schema2;
- private static Tuple tuple1;
- private static Tuple tuple2;
- private static final String schemaV1 = "{\"type\":\"record\","
- + "\"name\":\"myrecord\","
- + "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"},"
- + "{ \"name\":\"int1\", \"type\":\"int\" }]}";
-
- private static final String schemaV2 = "{\"type\":\"record\","
- + "\"name\":\"myrecord\","
- + "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"},"
- + "{ \"name\":\"bar\", \"type\":\"string\", \"default\":\"baz\" },"
- + "{ \"name\":\"int1\", \"type\":\"int\" }]}";
@BeforeClass
public static void setupClass() {
@@ -119,6 +108,18 @@ public class AvroGenericRecordBoltTest {
tuple2 = generateTestTuple(builder2.build());
}
+ private static Tuple generateTestTuple(GenericRecord record) {
+ TopologyBuilder builder = new TopologyBuilder();
+ GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(),
+ new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
+ @Override
+ public Fields getComponentOutputFields(String componentId, String streamId) {
+ return new Fields("record");
+ }
+ };
+ return new TupleImpl(topologyContext, new Values(record), topologyContext.getComponentId(1), 1, "");
+ }
+
@Before
public void setup() throws Exception {
fs = dfsClusterRule.getDfscluster().getFileSystem();
@@ -220,18 +221,6 @@ public class AvroGenericRecordBoltTest {
.withSyncPolicy(fieldsSyncPolicy);
}
- private static Tuple generateTestTuple(GenericRecord record) {
- TopologyBuilder builder = new TopologyBuilder();
- GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(),
- new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
- @Override
- public Fields getComponentOutputFields(String componentId, String streamId) {
- return new Fields("record");
- }
- };
- return new TupleImpl(topologyContext, new Values(record), topologyContext.getComponentId(1), 1, "");
- }
-
private void verifyAllAvroFiles(String path) throws IOException {
Path p = new Path(path);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java
index b872029..32844e7 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java
@@ -1,32 +1,29 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.storm.Config;
-import org.apache.storm.task.GeneralTopologyContext;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.TopologyBuilder;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.tuple.TupleImpl;
-import org.apache.storm.tuple.Values;
-import org.apache.storm.utils.MockTupleHelpers;
import org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat;
import org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
@@ -36,37 +33,33 @@ import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
import org.apache.storm.hdfs.common.Partitioner;
-import org.junit.Before;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.apache.storm.task.GeneralTopologyContext;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.TopologyBuilder;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.TupleImpl;
+import org.apache.storm.tuple.Values;
+import org.apache.storm.utils.MockTupleHelpers;
import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
-import org.junit.Assert;
-
import org.junit.rules.ExpectedException;
-import org.mockito.Mock;
-
-import static org.mockito.Mockito.*;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
import org.junit.runner.RunWith;
-import org.mockito.Mockito;
+import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+
@RunWith(MockitoJUnitRunner.class)
public class TestHdfsBolt {
+ private static final String testRoot = "/unittest";
@Rule
public MiniDFSClusterRule dfsClusterRule = new MiniDFSClusterRule(() -> {
Configuration conf = new Configuration();
@@ -77,16 +70,16 @@ public class TestHdfsBolt {
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
return conf;
});
-
- private String hdfsURI;
- private DistributedFileSystem fs;
- private static final String testRoot = "/unittest";
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
Tuple tuple1 = generateTestTuple(1, "First Tuple", "SFO", "CA");
Tuple tuple2 = generateTestTuple(1, "Second Tuple", "SJO", "CA");
-
- @Mock private OutputCollector collector;
- @Mock private TopologyContext topologyContext;
- @Rule public ExpectedException thrown = ExpectedException.none();
+ private String hdfsURI;
+ private DistributedFileSystem fs;
+ @Mock
+ private OutputCollector collector;
+ @Mock
+ private TopologyContext topologyContext;
@Before
public void setup() throws Exception {
@@ -138,8 +131,7 @@ public class TestHdfsBolt {
}
@Test
- public void testTwoTuplesOneFile() throws IOException
- {
+ public void testTwoTuplesOneFile() throws IOException {
HdfsBolt bolt = makeHdfsBolt(hdfsURI, 2, 10000f);
bolt.prepare(new Config(), topologyContext, collector);
bolt.execute(tuple1);
@@ -154,8 +146,7 @@ public class TestHdfsBolt {
}
@Test
- public void testFailedSync() throws IOException
- {
+ public void testFailedSync() throws IOException {
HdfsBolt bolt = makeHdfsBolt(hdfsURI, 2, 10000f);
bolt.prepare(new Config(), topologyContext, collector);
bolt.execute(tuple1);
@@ -171,27 +162,23 @@ public class TestHdfsBolt {
// One tuple and one rotation should yield one file with data
// The failed executions should not cause rotations and any new files
@Test
- public void testFailureFilecount() throws IOException, InterruptedException
- {
+ public void testFailureFilecount() throws IOException, InterruptedException {
HdfsBolt bolt = makeHdfsBolt(hdfsURI, 1, .000001f);
bolt.prepare(new Config(), topologyContext, collector);
bolt.execute(tuple1);
fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
- try
- {
+ try {
bolt.execute(tuple2);
} catch (RuntimeException e) {
//
}
- try
- {
+ try {
bolt.execute(tuple2);
} catch (RuntimeException e) {
//
}
- try
- {
+ try {
bolt.execute(tuple2);
} catch (RuntimeException e) {
//
@@ -202,8 +189,7 @@ public class TestHdfsBolt {
}
@Test
- public void testTickTuples() throws IOException
- {
+ public void testTickTuples() throws IOException {
HdfsBolt bolt = makeHdfsBolt(hdfsURI, 10, 10000f);
bolt.prepare(new Config(), topologyContext, collector);
@@ -230,28 +216,29 @@ public class TestHdfsBolt {
SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync);
FileRotationPolicy fieldsRotationPolicy =
- new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
+ new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
return new HdfsBolt()
- .withFsUrl(nameNodeAddr)
- .withFileNameFormat(fieldsFileNameFormat)
- .withRecordFormat(fieldsFormat)
- .withRotationPolicy(fieldsRotationPolicy)
- .withSyncPolicy(fieldsSyncPolicy);
+ .withFsUrl(nameNodeAddr)
+ .withFileNameFormat(fieldsFileNameFormat)
+ .withRecordFormat(fieldsFormat)
+ .withRotationPolicy(fieldsRotationPolicy)
+ .withSyncPolicy(fieldsSyncPolicy);
}
- private Tuple generateTestTuple(Object id, Object msg,Object city,Object state) {
+ private Tuple generateTestTuple(Object id, Object msg, Object city, Object state) {
TopologyBuilder builder = new TopologyBuilder();
GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(),
- new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") {
+ new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(),
+ "") {
@Override
public Fields getComponentOutputFields(String componentId, String streamId) {
- return new Fields("id", "msg","city","state");
+ return new Fields("id", "msg", "city", "state");
}
};
- return new TupleImpl(topologyContext, new Values(id, msg,city,state), topologyContext.getComponentId(1), 1, "");
+ return new TupleImpl(topologyContext, new Values(id, msg, city, state), topologyContext.getComponentId(1), 1, "");
}
// Generally used to compare how files were actually written and compare to expectations based on total
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java
index 71f5d4e..901f597 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java
@@ -1,26 +1,37 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.bolt;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyZeroInteractions;
+package org.apache.storm.hdfs.bolt;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.storm.Config;
+import org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat;
+import org.apache.storm.hdfs.bolt.format.DefaultSequenceFormat;
+import org.apache.storm.hdfs.bolt.format.FileNameFormat;
+import org.apache.storm.hdfs.bolt.format.SequenceFormat;
+import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
+import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
+import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
+import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
import org.apache.storm.task.GeneralTopologyContext;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -29,44 +40,29 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.TupleImpl;
import org.apache.storm.tuple.Values;
-import org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat;
-import org.apache.storm.hdfs.bolt.format.FileNameFormat;
-import org.apache.storm.hdfs.bolt.format.SequenceFormat;
-import org.apache.storm.hdfs.bolt.format.DefaultSequenceFormat;
-import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
-import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
-import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
-import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
-import org.junit.*;
-
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
import org.mockito.Mock;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.mockito.runners.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
-import org.junit.runner.RunWith;
-import org.mockito.runners.MockitoJUnitRunner;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
@RunWith(MockitoJUnitRunner.class)
public class TestSequenceFileBolt {
private static final Logger LOG = LoggerFactory.getLogger(TestSequenceFileBolt.class);
-
+ private static final String testRoot = "/unittest";
@Rule
public MiniDFSClusterRule dfsClusterRule = new MiniDFSClusterRule(() -> {
- Configuration conf = new Configuration();
+ Configuration conf = new Configuration();
conf.set("fs.trash.interval", "10");
conf.setBoolean("dfs.permissions", true);
File baseDir = new File("./target/hdfs/").getAbsoluteFile();
@@ -74,16 +70,16 @@ public class TestSequenceFileBolt {
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
return conf;
});
-
- private String hdfsURI;
- private DistributedFileSystem fs;
- private static final String testRoot = "/unittest";
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
Tuple tuple1 = generateTestTuple(1l, "first tuple");
Tuple tuple2 = generateTestTuple(2l, "second tuple");
-
- @Mock private OutputCollector collector;
- @Mock private TopologyContext topologyContext;
- @Rule public ExpectedException thrown = ExpectedException.none();
+ private String hdfsURI;
+ private DistributedFileSystem fs;
+ @Mock
+ private OutputCollector collector;
+ @Mock
+ private TopologyContext topologyContext;
@Before
public void setup() throws Exception {
@@ -111,8 +107,7 @@ public class TestSequenceFileBolt {
}
@Test
- public void testTwoTuplesOneFile() throws IOException
- {
+ public void testTwoTuplesOneFile() throws IOException {
SequenceFileBolt bolt = makeSeqBolt(hdfsURI, 2, 10000f);
bolt.prepare(new Config(), topologyContext, collector);
bolt.execute(tuple1);
@@ -127,8 +122,7 @@ public class TestSequenceFileBolt {
}
@Test
- public void testFailedSync() throws IOException
- {
+ public void testFailedSync() throws IOException {
SequenceFileBolt bolt = makeSeqBolt(hdfsURI, 2, 10000f);
bolt.prepare(new Config(), topologyContext, collector);
bolt.execute(tuple1);
@@ -144,24 +138,25 @@ public class TestSequenceFileBolt {
SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync);
FileRotationPolicy fieldsRotationPolicy =
- new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
+ new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
SequenceFormat seqFormat = new DefaultSequenceFormat("key", "value");
return new SequenceFileBolt()
- .withFsUrl(nameNodeAddr)
- .withFileNameFormat(fieldsFileNameFormat)
- .withRotationPolicy(fieldsRotationPolicy)
- .withSequenceFormat(seqFormat)
- .withSyncPolicy(fieldsSyncPolicy);
+ .withFsUrl(nameNodeAddr)
+ .withFileNameFormat(fieldsFileNameFormat)
+ .withRotationPolicy(fieldsRotationPolicy)
+ .withSequenceFormat(seqFormat)
+ .withSyncPolicy(fieldsSyncPolicy);
}
private Tuple generateTestTuple(Long key, String value) {
TopologyBuilder builder = new TopologyBuilder();
GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(),
- new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") {
+ new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(),
+ "") {
@Override
public Fields getComponentOutputFields(String componentId, String streamId) {
return new Fields("key", "value");
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestWritersMap.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestWritersMap.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestWritersMap.java
index 9554aba..2fecbe0 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestWritersMap.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestWritersMap.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.bolt;
@@ -34,8 +28,8 @@ public class TestWritersMap {
AbstractHDFSWriterMock bar = new AbstractHDFSWriterMock(new FileSizeRotationPolicy(1, FileSizeRotationPolicy.Units.KB), null);
AbstractHDFSWriterMock baz = new AbstractHDFSWriterMock(new FileSizeRotationPolicy(1, FileSizeRotationPolicy.Units.KB), null);
- @Test public void testLRUBehavior()
- {
+ @Test
+ public void testLRUBehavior() {
map.put("FOO", foo);
map.put("BAR", bar);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/format/TestSimpleFileNameFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/format/TestSimpleFileNameFormat.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/format/TestSimpleFileNameFormat.java
index 64f5dd8..fd586db 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/format/TestSimpleFileNameFormat.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/format/TestSimpleFileNameFormat.java
@@ -1,27 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt.format;
import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.storm.task.TopologyContext;
import org.apache.storm.utils.Utils;
import org.junit.Assert;
@@ -40,7 +34,7 @@ public class TestSimpleFileNameFormat {
Assert.assertEquals("/storm", path);
String time = new SimpleDateFormat("yyyyMMddHHmmss").format(now);
- Assert.assertEquals(time+".1.txt", name);
+ Assert.assertEquals(time + ".1.txt", name);
}
@Test
@@ -54,7 +48,7 @@ public class TestSimpleFileNameFormat {
long now = System.currentTimeMillis();
String path = format.getPath();
String name = format.getName(1, now);
-
+
Assert.assertEquals("/mypath", path);
String time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(now);
String host = null;
@@ -63,20 +57,21 @@ public class TestSimpleFileNameFormat {
} catch (UnknownHostException e) {
e.printStackTrace();
}
- Assert.assertEquals(time+"."+host+".Xcom.7.1.txt", name);
+ Assert.assertEquals(time + "." + host + ".Xcom.7.1.txt", name);
}
- @Test(expected=IllegalArgumentException.class)
+ @Test(expected = IllegalArgumentException.class)
public void testTimeFormat() {
Map<String, Object> topoConf = new HashMap();
SimpleFileNameFormat format = new SimpleFileNameFormat()
- .withTimeFormat("xyz");
+ .withTimeFormat("xyz");
format.prepare(null, createTopologyContext(topoConf));
}
-
- private TopologyContext createTopologyContext(Map<String, Object> topoConf){
- Map<Integer, String> taskToComponent = new HashMap<Integer, String>();
+
+ private TopologyContext createTopologyContext(Map<String, Object> topoConf) {
+ Map<Integer, String> taskToComponent = new HashMap<Integer, String>();
taskToComponent.put(7, "Xcom");
- return new TopologyContext(null, topoConf, taskToComponent, null, null, null, null, null, null, 7, 6703, null, null, null, null, null, null);
+ return new TopologyContext(null, topoConf, taskToComponent, null, null, null, null, null, null, 7, 6703, null, null, null, null,
+ null, null);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/ConfigsTest.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/ConfigsTest.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/ConfigsTest.java
index 6244466..c8c7620 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/ConfigsTest.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/ConfigsTest.java
@@ -1,32 +1,37 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.spout;
-import static org.junit.Assert.*;
+package org.apache.storm.hdfs.spout;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.storm.validation.ConfigValidation;
import org.junit.Test;
+import static org.junit.Assert.fail;
+
public class ConfigsTest {
+ public static void verifyBad(String key, Object value) {
+ Map<String, Object> conf = new HashMap<>();
+ conf.put(key, value);
+ try {
+ ConfigValidation.validateFields(conf);
+ fail("Expected " + key + " = " + value + " to throw Exception, but it didn't");
+ } catch (IllegalArgumentException e) {
+ //good
+ }
+ }
+
@SuppressWarnings("deprecation")
@Test
public void testGood() {
@@ -67,17 +72,6 @@ public class ConfigsTest {
ConfigValidation.validateFields(conf);
}
- public static void verifyBad(String key, Object value) {
- Map<String, Object> conf = new HashMap<>();
- conf.put(key, value);
- try {
- ConfigValidation.validateFields(conf);
- fail("Expected "+key+" = "+ value + " to throw Exception, but it didn't");
- } catch (IllegalArgumentException e) {
- //good
- }
- }
-
@SuppressWarnings("deprecation")
@Test
public void testBad() {
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java
index 4016c55..46f7892 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java
@@ -1,12 +1,7 @@
-
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
@@ -14,31 +9,29 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
+
package org.apache.storm.hdfs.spout;
+import java.io.IOException;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
import org.junit.Rule;
+import org.junit.Test;
public class TestDirLock {
+ private static final int LOCK_EXPIRY_SEC = 1;
+ private final Path locksDir = new Path("/tmp/lockdir");
@Rule
public MiniDFSClusterRule DFS_CLUSTER_RULE = new MiniDFSClusterRule();
-
- private static final int LOCK_EXPIRY_SEC = 1;
-
private FileSystem fs;
private HdfsConfiguration conf = new HdfsConfiguration();
- private final Path locksDir = new Path("/tmp/lockdir");
@Before
public void setUp() throws IOException {
@@ -92,7 +85,7 @@ public class TestDirLock {
for (DirLockingThread thread : threads) {
thread.interrupt();
thread.join(30_000);
- if(thread.isAlive()) {
+ if (thread.isAlive()) {
throw new RuntimeException("Failed to stop threads within 30 seconds, threads may leak into other tests");
}
}
@@ -134,10 +127,10 @@ public class TestDirLock {
class DirLockingThread extends Thread {
- private int thdNum;
private final FileSystem fs;
private final Path dir;
public boolean cleanExit = false;
+ private int thdNum;
public DirLockingThread(int thdNum, FileSystem fs, Path dir)
throws IOException {
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java
index df131a4..1a30bfa 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java
@@ -1,12 +1,7 @@
-
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
@@ -14,38 +9,42 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
+
package org.apache.storm.hdfs.spout;
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.storm.hdfs.common.HdfsUtils;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Test;
-
-import java.io.BufferedReader;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
import org.junit.Rule;
+import org.junit.Test;
public class TestFileLock {
+ private final Path filesDir = new Path("/tmp/filesdir");
+ private final Path locksDir = new Path("/tmp/locksdir");
@Rule
public MiniDFSClusterRule dfsClusterRule = new MiniDFSClusterRule();
-
private FileSystem fs;
private HdfsConfiguration conf = new HdfsConfiguration();
- private final Path filesDir = new Path("/tmp/filesdir");
- private final Path locksDir = new Path("/tmp/locksdir");
+ public static void closeUnderlyingLockFile(FileLock lock) throws ReflectiveOperationException {
+ Method m = FileLock.class.getDeclaredMethod("forceCloseLockFile");
+ m.setAccessible(true);
+ m.invoke(lock);
+ }
@Before
public void setup() throws IOException {
@@ -307,12 +306,6 @@ public class TestFileLock {
}
}
- public static void closeUnderlyingLockFile(FileLock lock) throws ReflectiveOperationException {
- Method m = FileLock.class.getDeclaredMethod("forceCloseLockFile");
- m.setAccessible(true);
- m.invoke(lock);
- }
-
/**
* return null if file not found
*/
@@ -340,9 +333,9 @@ public class TestFileLock {
class FileLockingThread extends Thread {
- private int thdNum;
private final FileSystem fs;
public boolean cleanExit = false;
+ private int thdNum;
private Path fileToLock;
private Path locksDir;
private String spoutId;
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java
index 8395d27..1a278c1 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java
@@ -1,12 +1,7 @@
-
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
@@ -14,12 +9,10 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
-package org.apache.storm.hdfs.spout;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.fail;
-import static org.mockito.ArgumentMatchers.notNull;
+package org.apache.storm.hdfs.spout;
+import java.io.IOException;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
@@ -28,24 +21,24 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
-import java.io.IOException;
-import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
-import org.junit.Rule;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.notNull;
public class TestHdfsSemantics {
+ private final HdfsConfiguration conf = new HdfsConfiguration();
+ private final Path dir = new Path("/tmp/filesdir");
@Rule
public MiniDFSClusterRule dfsClusterRule = new MiniDFSClusterRule();
-
private FileSystem fs;
- private final HdfsConfiguration conf = new HdfsConfiguration();
-
- private final Path dir = new Path("/tmp/filesdir");
@Before
public void setup() throws IOException {
@@ -63,7 +56,7 @@ public class TestHdfsSemantics {
@Test
public void testDeleteSemantics() throws Exception {
Path file = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1");
-// try {
+ // try {
// 1) Delete absent file - should return false
Assert.assertFalse(fs.exists(file));
try {
@@ -130,7 +123,7 @@ public class TestHdfsSemantics {
}
//2 try to append to a closed file
- try (FSDataOutputStream os2 = fs.append(file1)) {
+ try (FSDataOutputStream os2 = fs.append(file1)) {
assertThat(os2, notNull());
}
}
[29/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-hdfs
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java
index 53bcc4c..d4a95ee 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AbstractHDFSWriter.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.common;
@@ -25,11 +19,11 @@ import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
import org.apache.storm.tuple.Tuple;
abstract public class AbstractHDFSWriter implements Writer {
+ final protected Path filePath;
+ final protected FileRotationPolicy rotationPolicy;
protected long lastUsedTime;
protected long offset;
protected boolean needsRotation;
- final protected Path filePath;
- final protected FileRotationPolicy rotationPolicy;
public AbstractHDFSWriter(FileRotationPolicy policy, Path path) {
//This must be defensively copied, because a bolt probably has only one rotation policy object
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java
index 6e957c2..d77423c 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/AvroGenericRecordHDFSWriter.java
@@ -1,23 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.common;
+import java.io.IOException;
+import java.util.EnumSet;
import org.apache.avro.Schema;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericDatumWriter;
@@ -31,9 +28,6 @@ import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.EnumSet;
-
public class AvroGenericRecordHDFSWriter extends AbstractHDFSWriter {
private static final Logger LOG = LoggerFactory.getLogger(AvroGenericRecordHDFSWriter.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java
index d69d770..578bc06 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HDFSWriter.java
@@ -1,22 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.common;
+import java.io.IOException;
+import java.util.EnumSet;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
@@ -26,10 +23,7 @@ import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.EnumSet;
-
-public class HDFSWriter extends AbstractHDFSWriter{
+public class HDFSWriter extends AbstractHDFSWriter {
private static final Logger LOG = LoggerFactory.getLogger(HDFSWriter.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java
index 5ec5333..462087e 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/HdfsUtils.java
@@ -1,100 +1,93 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.common;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.ipc.RemoteException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-
public class HdfsUtils {
- /** list files sorted by modification time that have not been modified since 'olderThan'. if
- * 'olderThan' is <= 0 then the filtering is disabled */
- public static ArrayList<Path> listFilesByModificationTime(FileSystem fs, Path directory, long olderThan)
- throws IOException {
- ArrayList<LocatedFileStatus> fstats = new ArrayList<>();
+ /** list files sorted by modification time that have not been modified since 'olderThan'. if
+ * 'olderThan' is <= 0 then the filtering is disabled */
+ public static ArrayList<Path> listFilesByModificationTime(FileSystem fs, Path directory, long olderThan)
+ throws IOException {
+ ArrayList<LocatedFileStatus> fstats = new ArrayList<>();
- RemoteIterator<LocatedFileStatus> itr = fs.listFiles(directory, false);
- while( itr.hasNext() ) {
- LocatedFileStatus fileStatus = itr.next();
- if(olderThan>0) {
- if( fileStatus.getModificationTime()<=olderThan )
- fstats.add(fileStatus);
- }
- else {
- fstats.add(fileStatus);
- }
- }
- Collections.sort(fstats, new ModifTimeComparator() );
+ RemoteIterator<LocatedFileStatus> itr = fs.listFiles(directory, false);
+ while (itr.hasNext()) {
+ LocatedFileStatus fileStatus = itr.next();
+ if (olderThan > 0) {
+ if (fileStatus.getModificationTime() <= olderThan) {
+ fstats.add(fileStatus);
+ }
+ } else {
+ fstats.add(fileStatus);
+ }
+ }
+ Collections.sort(fstats, new ModifTimeComparator());
- ArrayList<Path> result = new ArrayList<>(fstats.size());
- for (LocatedFileStatus fstat : fstats) {
- result.add(fstat.getPath());
+ ArrayList<Path> result = new ArrayList<>(fstats.size());
+ for (LocatedFileStatus fstat : fstats) {
+ result.add(fstat.getPath());
+ }
+ return result;
}
- return result;
- }
- /**
- * Returns null if file already exists. throws if there was unexpected problem
- */
- public static FSDataOutputStream tryCreateFile(FileSystem fs, Path file) throws IOException {
- try {
- FSDataOutputStream os = fs.create(file, false);
- return os;
- } catch (FileAlreadyExistsException e) {
- return null;
- } catch (RemoteException e) {
- if( e.unwrapRemoteException() instanceof AlreadyBeingCreatedException ) {
- return null;
- } else { // unexpected error
- throw e;
- }
+ /**
+ * Returns null if file already exists. throws if there was unexpected problem
+ */
+ public static FSDataOutputStream tryCreateFile(FileSystem fs, Path file) throws IOException {
+ try {
+ FSDataOutputStream os = fs.create(file, false);
+ return os;
+ } catch (FileAlreadyExistsException e) {
+ return null;
+ } catch (RemoteException e) {
+ if (e.unwrapRemoteException() instanceof AlreadyBeingCreatedException) {
+ return null;
+ } else { // unexpected error
+ throw e;
+ }
+ }
}
- }
- public static class Pair<K,V> {
- private K key;
- private V value;
- public Pair(K key, V value) {
- this.key = key;
- this.value = value;
- }
+ public static class Pair<K, V> {
+ private K key;
+ private V value;
- public K getKey() {
- return key;
- }
+ public Pair(K key, V value) {
+ this.key = key;
+ this.value = value;
+ }
- public V getValue() {
- return value;
- }
+ public static <K, V> Pair of(K key, V value) {
+ return new Pair(key, value);
+ }
- public static <K,V> Pair of(K key, V value) {
- return new Pair(key,value);
- }
- } // class Pair
+ public K getKey() {
+ return key;
+ }
+
+ public V getValue() {
+ return value;
+ }
+ } // class Pair
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java
index 0558b3f..47ebdfe 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/ModifTimeComparator.java
@@ -1,32 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.common;
-import org.apache.hadoop.fs.FileStatus;
-
import java.util.Comparator;
+import org.apache.hadoop.fs.FileStatus;
public class ModifTimeComparator
- implements Comparator<FileStatus> {
- @Override
+ implements Comparator<FileStatus> {
+ @Override
public int compare(FileStatus o1, FileStatus o2) {
- return new Long(o1.getModificationTime()).compareTo( o2.getModificationTime() );
+ return new Long(o1.getModificationTime()).compareTo(o2.getModificationTime());
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java
index fd50496..3137f48 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/NullPartitioner.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.common;
import org.apache.storm.tuple.Tuple;
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java
index 6cf0fbd..9f79373 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/Partitioner.java
@@ -1,27 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.common;
-import org.apache.storm.tuple.Tuple;
+package org.apache.storm.hdfs.common;
import java.io.Serializable;
+import org.apache.storm.tuple.Tuple;
-public interface Partitioner extends Serializable{
+public interface Partitioner extends Serializable {
/**
* Return a relative path that the tuple should be written to. For example, if an HdfsBolt were configured to write
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java
index ec78fd6..d0507b8 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/SequenceFileWriter.java
@@ -1,22 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.common;
+import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.storm.hdfs.bolt.format.SequenceFormat;
@@ -25,9 +21,7 @@ import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-
-public class SequenceFileWriter extends AbstractHDFSWriter{
+public class SequenceFileWriter extends AbstractHDFSWriter {
private static final Logger LOG = LoggerFactory.getLogger(SequenceFileWriter.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/rotation/MoveFileAction.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/rotation/MoveFileAction.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/rotation/MoveFileAction.java
index 585307d..0e8e1dd 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/rotation/MoveFileAction.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/rotation/MoveFileAction.java
@@ -15,21 +15,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.common.rotation;
+import java.io.IOException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-
public class MoveFileAction implements RotationAction {
private static final Logger LOG = LoggerFactory.getLogger(MoveFileAction.class);
private String destination;
- public MoveFileAction toDestination(String destDir){
+ public MoveFileAction toDestination(String destDir) {
destination = destDir;
return this;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/rotation/RotationAction.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/rotation/RotationAction.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/rotation/RotationAction.java
index b15c314..f5ade03 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/rotation/RotationAction.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/common/rotation/RotationAction.java
@@ -15,14 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.hdfs.common.rotation;
+package org.apache.storm.hdfs.common.rotation;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import java.io.IOException;
import java.io.Serializable;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
public interface RotationAction extends Serializable {
void execute(FileSystem fileSystem, Path filePath) throws IOException;
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java
index e1339df..6d5537b 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/AbstractFileReader.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
@@ -24,37 +18,41 @@ import org.apache.hadoop.fs.Path;
abstract class AbstractFileReader implements FileReader {
- private final Path file;
+ private final Path file;
- public AbstractFileReader(FileSystem fs, Path file) {
- if (fs == null ) {
- throw new IllegalArgumentException("filesystem arg cannot be null for reader");
+ public AbstractFileReader(FileSystem fs, Path file) {
+ if (fs == null) {
+ throw new IllegalArgumentException("filesystem arg cannot be null for reader");
+ }
+ if (file == null) {
+ throw new IllegalArgumentException("file arg cannot be null for reader");
+ }
+ this.file = file;
}
- if (file == null ) {
- throw new IllegalArgumentException("file arg cannot be null for reader");
- }
- this.file = file;
- }
- @Override
- public Path getFilePath() {
- return file;
- }
+ @Override
+ public Path getFilePath() {
+ return file;
+ }
- @Override
- public boolean equals(Object o) {
- if (this == o) { return true; }
- if (o == null || getClass() != o.getClass()) { return false; }
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
- AbstractFileReader that = (AbstractFileReader) o;
+ AbstractFileReader that = (AbstractFileReader) o;
- return !(file != null ? !file.equals(that.file) : that.file != null);
- }
+ return !(file != null ? !file.equals(that.file) : that.file != null);
+ }
- @Override
- public int hashCode() {
- return file != null ? file.hashCode() : 0;
- }
+ @Override
+ public int hashCode() {
+ return file != null ? file.hashCode() : 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java
index cb2607a..f94b8e5 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/Configs.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
@@ -29,13 +23,6 @@ import org.apache.storm.validation.NotConf;
import org.apache.storm.validation.Validated;
public class Configs implements Validated {
- public static class ReaderTypeValidator extends Validator {
- @Override
- public void validateField(String name, Object o) {
- HdfsSpout.checkValidReader((String)o);
- }
- }
-
/**
* @deprecated please use {@link HdfsSpout.setReaderType(String)}
*/
@@ -45,7 +32,6 @@ public class Configs implements Validated {
public static final String READER_TYPE = "hdfsspout.reader.type"; // Required - chose the file type being consumed
public static final String TEXT = "text";
public static final String SEQ = "seq";
-
/**
* @deprecated please use {@link HdfsSpout#setHdfsUri(String)}
*/
@@ -81,7 +67,7 @@ public class Configs implements Validated {
*/
@Deprecated
@isInteger
- @isPositiveNumber(includeZero=true)
+ @isPositiveNumber(includeZero = true)
public static final String COMMIT_FREQ_COUNT = "hdfsspout.commit.count"; // commit after N records. 0 disables this.
/**
* @deprecated please use {@link HdfsSpout#setCommitFrequencySec(int)}
@@ -95,7 +81,7 @@ public class Configs implements Validated {
*/
@Deprecated
@isInteger
- @isPositiveNumber(includeZero=true)
+ @isPositiveNumber(includeZero = true)
public static final String MAX_OUTSTANDING = "hdfsspout.max.outstanding";
/**
* @deprecated please use {@link HdfsSpout#setLockTimeoutSec(int)}
@@ -103,27 +89,34 @@ public class Configs implements Validated {
@Deprecated
@isInteger
@isPositiveNumber
- public static final String LOCK_TIMEOUT = "hdfsspout.lock.timeout.sec"; // inactivity duration after which locks are considered candidates for being reassigned to another spout
+ public static final String LOCK_TIMEOUT = "hdfsspout.lock.timeout.sec";
/**
* @deprecated please use {@link HdfsSpout#setClocksInSync(boolean)}
*/
@Deprecated
@isBoolean
public static final String CLOCKS_INSYNC = "hdfsspout.clocks.insync"; // if clocks on machines in the Storm cluster are in sync
+ // inactivity duration after which locks are considered candidates for being reassigned to another spout
/**
* @deprecated please use {@link HdfsSpout#setIgnoreSuffix(String)}
*/
@Deprecated
@isString
- public static final String IGNORE_SUFFIX = "hdfsspout.ignore.suffix"; // filenames with this suffix in archive dir will be ignored by the Spout
-
+ public static final String IGNORE_SUFFIX = "hdfsspout.ignore.suffix";
@NotConf
public static final String DEFAULT_LOCK_DIR = ".lock";
+ // filenames with this suffix in archive dir will be ignored by the Spout
public static final int DEFAULT_COMMIT_FREQ_COUNT = 20000;
public static final int DEFAULT_COMMIT_FREQ_SEC = 10;
public static final int DEFAULT_MAX_OUTSTANDING = 10000;
public static final int DEFAULT_LOCK_TIMEOUT = 5 * 60; // 5 min
-
@isMapEntryType(keyType = String.class, valueType = String.class)
public static final String DEFAULT_HDFS_CONFIG_KEY = "hdfs.config";
+
+ public static class ReaderTypeValidator extends Validator {
+ @Override
+ public void validateField(String name, Object o) {
+ HdfsSpout.checkValidReader((String) o);
+ }
+ }
} // class Configs
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java
index 25a136c..eea23e1 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/DirLock.java
@@ -1,23 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
+import java.io.IOException;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -26,108 +21,107 @@ import org.apache.storm.hdfs.common.HdfsUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-
/**
* Facility to synchronize access to HDFS directory. The lock itself is represented
* as a file in the same directory. Relies on atomic file creation.
*/
public class DirLock {
- private FileSystem fs;
- private final Path lockFile;
- public static final String DIR_LOCK_FILE = "DIRLOCK";
- private static final Logger LOG = LoggerFactory.getLogger(DirLock.class);
- private DirLock(FileSystem fs, Path lockFile) throws IOException {
- if( fs.isDirectory(lockFile) ) {
- throw new IllegalArgumentException(lockFile.toString() + " is not a directory");
- }
- this.fs = fs;
- this.lockFile = lockFile;
- }
-
- /** Get a lock on file if not already locked
- *
- * @param fs
- * @param dir the dir on which to get a lock
- * @return The lock object if it the lock was acquired. Returns null if the dir is already locked.
- * @throws IOException if there were errors
- */
- public static DirLock tryLock(FileSystem fs, Path dir) throws IOException {
- Path lockFile = getDirLockFile(dir);
+ public static final String DIR_LOCK_FILE = "DIRLOCK";
+ private static final Logger LOG = LoggerFactory.getLogger(DirLock.class);
+ private final Path lockFile;
+ private FileSystem fs;
- try {
- FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, lockFile);
- if (ostream!=null) {
- LOG.debug("Thread ({}) Acquired lock on dir {}", threadInfo(), dir);
- ostream.close();
- return new DirLock(fs, lockFile);
- } else {
- LOG.debug("Thread ({}) cannot lock dir {} as its already locked.", threadInfo(), dir);
- return null;
- }
- } catch (IOException e) {
- LOG.error("Error when acquiring lock on dir " + dir, e);
- throw e;
+ private DirLock(FileSystem fs, Path lockFile) throws IOException {
+ if (fs.isDirectory(lockFile)) {
+ throw new IllegalArgumentException(lockFile.toString() + " is not a directory");
+ }
+ this.fs = fs;
+ this.lockFile = lockFile;
}
- }
- private static Path getDirLockFile(Path dir) {
- return new Path(dir.toString() + Path.SEPARATOR_CHAR + DIR_LOCK_FILE );
- }
+ /** Get a lock on file if not already locked
+ *
+ * @param fs
+ * @param dir the dir on which to get a lock
+ * @return The lock object if it the lock was acquired. Returns null if the dir is already locked.
+ * @throws IOException if there were errors
+ */
+ public static DirLock tryLock(FileSystem fs, Path dir) throws IOException {
+ Path lockFile = getDirLockFile(dir);
- private static String threadInfo () {
- return "ThdId=" + Thread.currentThread().getId() + ", ThdName="
- + Thread.currentThread().getName();
- }
+ try {
+ FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, lockFile);
+ if (ostream != null) {
+ LOG.debug("Thread ({}) Acquired lock on dir {}", threadInfo(), dir);
+ ostream.close();
+ return new DirLock(fs, lockFile);
+ } else {
+ LOG.debug("Thread ({}) cannot lock dir {} as its already locked.", threadInfo(), dir);
+ return null;
+ }
+ } catch (IOException e) {
+ LOG.error("Error when acquiring lock on dir " + dir, e);
+ throw e;
+ }
+ }
- /** Release lock on dir by deleting the lock file */
- public void release() throws IOException {
- if(!fs.delete(lockFile, false)) {
- LOG.error("Thread {} could not delete dir lock {} ", threadInfo(), lockFile);
+ private static Path getDirLockFile(Path dir) {
+ return new Path(dir.toString() + Path.SEPARATOR_CHAR + DIR_LOCK_FILE);
}
- else {
- LOG.debug("Thread {} Released dir lock {} ", threadInfo(), lockFile);
+
+ private static String threadInfo() {
+ return "ThdId=" + Thread.currentThread().getId() + ", ThdName="
+ + Thread.currentThread().getName();
}
- }
- /** if the lock on the directory is stale, take ownership */
- public static DirLock takeOwnershipIfStale(FileSystem fs, Path dirToLock, int lockTimeoutSec) {
- Path dirLockFile = getDirLockFile(dirToLock);
+ /** if the lock on the directory is stale, take ownership */
+ public static DirLock takeOwnershipIfStale(FileSystem fs, Path dirToLock, int lockTimeoutSec) {
+ Path dirLockFile = getDirLockFile(dirToLock);
- long now = System.currentTimeMillis();
- long expiryTime = now - (lockTimeoutSec*1000);
+ long now = System.currentTimeMillis();
+ long expiryTime = now - (lockTimeoutSec * 1000);
- try {
- long modTime = fs.getFileStatus(dirLockFile).getModificationTime();
- if(modTime <= expiryTime) {
- return takeOwnership(fs, dirLockFile);
- }
- return null;
- } catch (IOException e) {
- return null;
+ try {
+ long modTime = fs.getFileStatus(dirLockFile).getModificationTime();
+ if (modTime <= expiryTime) {
+ return takeOwnership(fs, dirLockFile);
+ }
+ return null;
+ } catch (IOException e) {
+ return null;
+ }
}
- }
- private static DirLock takeOwnership(FileSystem fs, Path dirLockFile) throws IOException {
- if(fs instanceof DistributedFileSystem) {
- if (!((DistributedFileSystem) fs).recoverLease(dirLockFile)) {
- LOG.warn("Unable to recover lease on dir lock file " + dirLockFile + " right now. Cannot transfer ownership. Will need to try later.");
+ private static DirLock takeOwnership(FileSystem fs, Path dirLockFile) throws IOException {
+ if (fs instanceof DistributedFileSystem) {
+ if (!((DistributedFileSystem) fs).recoverLease(dirLockFile)) {
+ LOG.warn("Unable to recover lease on dir lock file " + dirLockFile +
+ " right now. Cannot transfer ownership. Will need to try later.");
+ return null;
+ }
+ }
+
+ // delete and recreate lock file
+ if (fs.delete(dirLockFile, false)) { // returns false if somebody else already deleted it (to take ownership)
+ FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, dirLockFile);
+ if (ostream != null) {
+ ostream.close();
+ }
+ return new DirLock(fs, dirLockFile);
+ }
return null;
- }
}
- // delete and recreate lock file
- if( fs.delete(dirLockFile, false) ) { // returns false if somebody else already deleted it (to take ownership)
- FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, dirLockFile);
- if(ostream!=null) {
- ostream.close();
- }
- return new DirLock(fs, dirLockFile);
+ /** Release lock on dir by deleting the lock file */
+ public void release() throws IOException {
+ if (!fs.delete(lockFile, false)) {
+ LOG.error("Thread {} could not delete dir lock {} ", threadInfo(), lockFile);
+ } else {
+ LOG.debug("Thread {} Released dir lock {} ", threadInfo(), lockFile);
+ }
}
- return null;
- }
- public Path getLockFile() {
- return lockFile;
- }
+ public Path getLockFile() {
+ return lockFile;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java
index a7cb2b8..c5a2f55 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileLock.java
@@ -1,24 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Collection;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -30,11 +28,6 @@ import org.apache.storm.hdfs.common.HdfsUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.Collection;
-
/**
* Facility to synchronize access to HDFS files. Thread gains exclusive access to a file by acquiring
* a FileLock object. The lock itself is represented as file on HDFS. Relies on atomic file creation.
@@ -43,291 +36,303 @@ import java.util.Collection;
*/
public class FileLock {
- private final FileSystem fs;
- private final String componentID;
- private final Path lockFile;
- private final FSDataOutputStream lockFileStream;
- private LogEntry lastEntry;
-
- private static final Logger LOG = LoggerFactory.getLogger(FileLock.class);
-
- private FileLock(FileSystem fs, Path lockFile, FSDataOutputStream lockFileStream, String spoutId)
- throws IOException {
- this.fs = fs;
- this.lockFile = lockFile;
- this.lockFileStream = lockFileStream;
- this.componentID = spoutId;
- logProgress("0", false);
- }
-
- private FileLock(FileSystem fs, Path lockFile, String spoutId, LogEntry entry)
- throws IOException {
- this.fs = fs;
- this.lockFile = lockFile;
- this.lockFileStream = fs.append(lockFile);
- this.componentID = spoutId;
- LOG.info("Acquired abandoned lockFile {}, Spout {}", lockFile, spoutId);
- logProgress(entry.fileOffset, true);
- }
-
- public void heartbeat(String fileOffset) throws IOException {
- logProgress(fileOffset, true);
- }
-
- // new line is at beginning of each line (instead of end) for better recovery from
- // partial writes of prior lines
- private void logProgress(String fileOffset, boolean prefixNewLine)
- throws IOException {
- long now = System.currentTimeMillis();
- LogEntry entry = new LogEntry(now, componentID, fileOffset);
- String line = entry.toString();
- if(prefixNewLine) {
- lockFileStream.writeBytes(System.lineSeparator() + line);
+ private static final Logger LOG = LoggerFactory.getLogger(FileLock.class);
+ private final FileSystem fs;
+ private final String componentID;
+ private final Path lockFile;
+ private final FSDataOutputStream lockFileStream;
+ private LogEntry lastEntry;
+
+ private FileLock(FileSystem fs, Path lockFile, FSDataOutputStream lockFileStream, String spoutId)
+ throws IOException {
+ this.fs = fs;
+ this.lockFile = lockFile;
+ this.lockFileStream = lockFileStream;
+ this.componentID = spoutId;
+ logProgress("0", false);
}
- else {
- lockFileStream.writeBytes(line);
+
+ private FileLock(FileSystem fs, Path lockFile, String spoutId, LogEntry entry)
+ throws IOException {
+ this.fs = fs;
+ this.lockFile = lockFile;
+ this.lockFileStream = fs.append(lockFile);
+ this.componentID = spoutId;
+ LOG.info("Acquired abandoned lockFile {}, Spout {}", lockFile, spoutId);
+ logProgress(entry.fileOffset, true);
}
- lockFileStream.hflush();
-
- lastEntry = entry; // update this only after writing to hdfs
- }
-
- /** Release lock by deleting file
- * @throws IOException if lock file could not be deleted
- */
- public void release() throws IOException {
- lockFileStream.close();
- if(!fs.delete(lockFile, false)) {
- LOG.warn("Unable to delete lock file, Spout = {}", componentID);
- throw new IOException("Unable to delete lock file");
+
+ /** returns lock on file or null if file is already locked. throws if unexpected problem */
+ public static FileLock tryLock(FileSystem fs, Path fileToLock, Path lockDirPath, String spoutId)
+ throws IOException {
+ Path lockFile = new Path(lockDirPath, fileToLock.getName());
+
+ try {
+ FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, lockFile);
+ if (ostream != null) {
+ LOG.debug("Acquired lock on file {}. LockFile= {}, Spout = {}", fileToLock, lockFile, spoutId);
+ return new FileLock(fs, lockFile, ostream, spoutId);
+ } else {
+ LOG.debug("Cannot lock file {} as its already locked. Spout = {}", fileToLock, spoutId);
+ return null;
+ }
+ } catch (IOException e) {
+ LOG.error("Error when acquiring lock on file " + fileToLock + " Spout = " + spoutId, e);
+ throw e;
+ }
}
- LOG.debug("Released lock file {}. Spout {}", lockFile, componentID);
- }
-
- // For testing only.. invoked via reflection
- private void forceCloseLockFile() throws IOException {
- lockFileStream.close();
- }
-
- /** returns lock on file or null if file is already locked. throws if unexpected problem */
- public static FileLock tryLock(FileSystem fs, Path fileToLock, Path lockDirPath, String spoutId)
- throws IOException {
- Path lockFile = new Path(lockDirPath, fileToLock.getName());
-
- try {
- FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, lockFile);
- if (ostream != null) {
- LOG.debug("Acquired lock on file {}. LockFile= {}, Spout = {}", fileToLock, lockFile, spoutId);
- return new FileLock(fs, lockFile, ostream, spoutId);
- } else {
- LOG.debug("Cannot lock file {} as its already locked. Spout = {}", fileToLock, spoutId);
+
+ /**
+ * checks if lockFile is older than 'olderThan' UTC time by examining the modification time
+ * on file and (if necessary) the timestamp in last log entry in the file. If its stale, then
+ * returns the last log entry, else returns null.
+ * @param fs
+ * @param lockFile
+ * @param olderThan time (millis) in UTC.
+ * @return the last entry in the file if its too old. null if last entry is not too old
+ * @throws IOException
+ */
+ public static LogEntry getLastEntryIfStale(FileSystem fs, Path lockFile, long olderThan)
+ throws IOException {
+ long modifiedTime = fs.getFileStatus(lockFile).getModificationTime();
+ if (modifiedTime <= olderThan) { // look
+ //Impt: HDFS timestamp may not reflect recent appends, so we double check the
+ // timestamp in last line of file to see when the last update was made
+ LogEntry lastEntry = getLastEntry(fs, lockFile);
+ if (lastEntry == null) {
+ LOG.warn("Empty lock file found. Deleting it. {}", lockFile);
+ try {
+ if (!fs.delete(lockFile, false)) {
+ throw new IOException("Empty lock file deletion failed");
+ }
+ } catch (Exception e) {
+ LOG.error("Unable to delete empty lock file " + lockFile, e);
+ }
+ }
+ if (lastEntry.eventTime <= olderThan) {
+ return lastEntry;
+ }
+ }
return null;
- }
- } catch (IOException e) {
- LOG.error("Error when acquiring lock on file " + fileToLock + " Spout = " + spoutId, e);
- throw e;
}
- }
-
- /**
- * checks if lockFile is older than 'olderThan' UTC time by examining the modification time
- * on file and (if necessary) the timestamp in last log entry in the file. If its stale, then
- * returns the last log entry, else returns null.
- * @param fs
- * @param lockFile
- * @param olderThan time (millis) in UTC.
- * @return the last entry in the file if its too old. null if last entry is not too old
- * @throws IOException
- */
- public static LogEntry getLastEntryIfStale(FileSystem fs, Path lockFile, long olderThan)
- throws IOException {
- long modifiedTime = fs.getFileStatus(lockFile).getModificationTime();
- if( modifiedTime <= olderThan ) { // look
- //Impt: HDFS timestamp may not reflect recent appends, so we double check the
- // timestamp in last line of file to see when the last update was made
- LogEntry lastEntry = getLastEntry(fs, lockFile);
- if(lastEntry==null) {
- LOG.warn("Empty lock file found. Deleting it. {}", lockFile);
- try {
- if(!fs.delete(lockFile, false))
- throw new IOException("Empty lock file deletion failed");
- } catch (Exception e) {
- LOG.error("Unable to delete empty lock file " + lockFile, e);
+
+ /**
+ * returns the last log entry
+ * @param fs
+ * @param lockFile
+ * @return
+ * @throws IOException
+ */
+ public static LogEntry getLastEntry(FileSystem fs, Path lockFile)
+ throws IOException {
+ FSDataInputStream in = fs.open(lockFile);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ String lastLine = null;
+ for (String line = reader.readLine(); line != null; line = reader.readLine()) {
+ lastLine = line;
}
- }
- if( lastEntry.eventTime <= olderThan )
- return lastEntry;
+ return LogEntry.deserialize(lastLine);
}
- return null;
- }
-
- /**
- * returns the last log entry
- * @param fs
- * @param lockFile
- * @return
- * @throws IOException
- */
- public static LogEntry getLastEntry(FileSystem fs, Path lockFile)
- throws IOException {
- FSDataInputStream in = fs.open(lockFile);
- BufferedReader reader = new BufferedReader(new InputStreamReader(in));
- String lastLine = null;
- for(String line = reader.readLine(); line!=null; line = reader.readLine() ) {
- lastLine=line;
+
+ /**
+ * Takes ownership of the lock file if possible.
+ * @param lockFile
+ * @param lastEntry last entry in the lock file. this param is an optimization.
+ * we dont scan the lock file again to find its last entry here since
+ * its already been done once by the logic used to check if the lock
+ * file is stale. so this value comes from that earlier scan.
+ * @param spoutId spout id
+ * @throws IOException if unable to acquire
+ * @return null if lock File is not recoverable
+ */
+ public static FileLock takeOwnership(FileSystem fs, Path lockFile, LogEntry lastEntry, String spoutId)
+ throws IOException {
+ try {
+ if (fs instanceof DistributedFileSystem) {
+ if (!((DistributedFileSystem) fs).recoverLease(lockFile)) {
+ LOG.warn(
+ "Unable to recover lease on lock file {} right now. Cannot transfer ownership. Will need to try later. Spout = {}",
+ lockFile, spoutId);
+ return null;
+ }
+ }
+ return new FileLock(fs, lockFile, spoutId, lastEntry);
+ } catch (IOException e) {
+ if (e instanceof RemoteException &&
+ ((RemoteException) e).unwrapRemoteException() instanceof AlreadyBeingCreatedException) {
+ LOG.warn(
+ "Lock file " + lockFile + "is currently open. Cannot transfer ownership now. Will need to try later. Spout= " + spoutId,
+ e);
+ return null;
+ } else { // unexpected error
+ LOG.warn("Cannot transfer ownership now for lock file " + lockFile + ". Will need to try later. Spout =" + spoutId, e);
+ throw e;
+ }
+ }
}
- return LogEntry.deserialize(lastLine);
- }
-
- /**
- * Takes ownership of the lock file if possible.
- * @param lockFile
- * @param lastEntry last entry in the lock file. this param is an optimization.
- * we dont scan the lock file again to find its last entry here since
- * its already been done once by the logic used to check if the lock
- * file is stale. so this value comes from that earlier scan.
- * @param spoutId spout id
- * @throws IOException if unable to acquire
- * @return null if lock File is not recoverable
- */
- public static FileLock takeOwnership(FileSystem fs, Path lockFile, LogEntry lastEntry, String spoutId)
- throws IOException {
- try {
- if(fs instanceof DistributedFileSystem ) {
- if( !((DistributedFileSystem) fs).recoverLease(lockFile) ) {
- LOG.warn("Unable to recover lease on lock file {} right now. Cannot transfer ownership. Will need to try later. Spout = {}", lockFile, spoutId);
- return null;
+
+ /**
+ * Finds a oldest expired lock file (using modification timestamp), then takes
+ * ownership of the lock file
+ * Impt: Assumes access to lockFilesDir has been externally synchronized such that
+ * only one thread accessing the same thread
+ * @param fs
+ * @param lockFilesDir
+ * @param locktimeoutSec
+ * @return
+ */
+ public static FileLock acquireOldestExpiredLock(FileSystem fs, Path lockFilesDir, int locktimeoutSec, String spoutId)
+ throws IOException {
+ // list files
+ long now = System.currentTimeMillis();
+ long olderThan = now - (locktimeoutSec * 1000);
+ Collection<Path> listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, olderThan);
+
+ // locate expired lock files (if any). Try to take ownership (oldest lock first)
+ for (Path file : listing) {
+ if (file.getName().equalsIgnoreCase(DirLock.DIR_LOCK_FILE)) {
+ continue;
+ }
+ LogEntry lastEntry = getLastEntryIfStale(fs, file, olderThan);
+ if (lastEntry != null) {
+ FileLock lock = FileLock.takeOwnership(fs, file, lastEntry, spoutId);
+ if (lock != null) {
+ return lock;
+ }
+ }
+ }
+ if (listing.isEmpty()) {
+ LOG.debug("No abandoned lock files found by Spout {}", spoutId);
}
- }
- return new FileLock(fs, lockFile, spoutId, lastEntry);
- } catch (IOException e) {
- if (e instanceof RemoteException &&
- ((RemoteException) e).unwrapRemoteException() instanceof AlreadyBeingCreatedException) {
- LOG.warn("Lock file " + lockFile + "is currently open. Cannot transfer ownership now. Will need to try later. Spout= " + spoutId, e);
return null;
- } else { // unexpected error
- LOG.warn("Cannot transfer ownership now for lock file " + lockFile + ". Will need to try later. Spout =" + spoutId, e);
- throw e;
- }
}
- }
-
- /**
- * Finds a oldest expired lock file (using modification timestamp), then takes
- * ownership of the lock file
- * Impt: Assumes access to lockFilesDir has been externally synchronized such that
- * only one thread accessing the same thread
- * @param fs
- * @param lockFilesDir
- * @param locktimeoutSec
- * @return
- */
- public static FileLock acquireOldestExpiredLock(FileSystem fs, Path lockFilesDir, int locktimeoutSec, String spoutId)
- throws IOException {
- // list files
- long now = System.currentTimeMillis();
- long olderThan = now - (locktimeoutSec*1000);
- Collection<Path> listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, olderThan);
-
- // locate expired lock files (if any). Try to take ownership (oldest lock first)
- for (Path file : listing) {
- if(file.getName().equalsIgnoreCase( DirLock.DIR_LOCK_FILE) ) {
- continue;
- }
- LogEntry lastEntry = getLastEntryIfStale(fs, file, olderThan);
- if(lastEntry!=null) {
- FileLock lock = FileLock.takeOwnership(fs, file, lastEntry, spoutId);
- if(lock!=null) {
- return lock;
+
+ /**
+ * Finds oldest expired lock file (using modification timestamp), then takes
+ * ownership of the lock file
+ * Impt: Assumes access to lockFilesDir has been externally synchronized such that
+ * only one thread accessing the same thread
+ * @param fs
+ * @param lockFilesDir
+ * @param locktimeoutSec
+ * @return a Pair<lock file path, last entry in lock file> .. if expired lock file found
+ * @throws IOException
+ */
+ public static HdfsUtils.Pair<Path, LogEntry> locateOldestExpiredLock(FileSystem fs, Path lockFilesDir, int locktimeoutSec)
+ throws IOException {
+ // list files
+ long now = System.currentTimeMillis();
+ long olderThan = now - (locktimeoutSec * 1000);
+ Collection<Path> listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, olderThan);
+
+ // locate oldest expired lock file (if any) and take ownership
+ for (Path file : listing) {
+ if (file.getName().equalsIgnoreCase(DirLock.DIR_LOCK_FILE)) {
+ continue;
+ }
+ LogEntry lastEntry = getLastEntryIfStale(fs, file, olderThan);
+ if (lastEntry != null) {
+ return new HdfsUtils.Pair<>(file, lastEntry);
+ }
}
- }
+ LOG.debug("No abandoned files found");
+ return null;
}
- if(listing.isEmpty()) {
- LOG.debug("No abandoned lock files found by Spout {}", spoutId);
+
+ public void heartbeat(String fileOffset) throws IOException {
+ logProgress(fileOffset, true);
}
- return null;
- }
-
-
- /**
- * Finds oldest expired lock file (using modification timestamp), then takes
- * ownership of the lock file
- * Impt: Assumes access to lockFilesDir has been externally synchronized such that
- * only one thread accessing the same thread
- * @param fs
- * @param lockFilesDir
- * @param locktimeoutSec
- * @return a Pair<lock file path, last entry in lock file> .. if expired lock file found
- * @throws IOException
- */
- public static HdfsUtils.Pair<Path,LogEntry> locateOldestExpiredLock(FileSystem fs, Path lockFilesDir, int locktimeoutSec)
- throws IOException {
- // list files
- long now = System.currentTimeMillis();
- long olderThan = now - (locktimeoutSec*1000);
- Collection<Path> listing = HdfsUtils.listFilesByModificationTime(fs, lockFilesDir, olderThan);
-
- // locate oldest expired lock file (if any) and take ownership
- for (Path file : listing) {
- if(file.getName().equalsIgnoreCase( DirLock.DIR_LOCK_FILE) ) {
- continue;
- }
- LogEntry lastEntry = getLastEntryIfStale(fs, file, olderThan);
- if(lastEntry!=null) {
- return new HdfsUtils.Pair<>(file, lastEntry);
- }
+
+ // new line is at beginning of each line (instead of end) for better recovery from
+ // partial writes of prior lines
+ private void logProgress(String fileOffset, boolean prefixNewLine)
+ throws IOException {
+ long now = System.currentTimeMillis();
+ LogEntry entry = new LogEntry(now, componentID, fileOffset);
+ String line = entry.toString();
+ if (prefixNewLine) {
+ lockFileStream.writeBytes(System.lineSeparator() + line);
+ } else {
+ lockFileStream.writeBytes(line);
+ }
+ lockFileStream.hflush();
+
+ lastEntry = entry; // update this only after writing to hdfs
+ }
+
+ /** Release lock by deleting file
+ * @throws IOException if lock file could not be deleted
+ */
+ public void release() throws IOException {
+ lockFileStream.close();
+ if (!fs.delete(lockFile, false)) {
+ LOG.warn("Unable to delete lock file, Spout = {}", componentID);
+ throw new IOException("Unable to delete lock file");
+ }
+ LOG.debug("Released lock file {}. Spout {}", lockFile, componentID);
}
- LOG.debug("No abandoned files found");
- return null;
- }
-
- public LogEntry getLastLogEntry() {
- return lastEntry;
- }
-
- public Path getLockFile() {
- return lockFile;
- }
-
- public static class LogEntry {
- private static final int NUM_FIELDS = 3;
- public final long eventTime;
- public final String componentID;
- public final String fileOffset;
-
- public LogEntry(long eventtime, String componentID, String fileOffset) {
- this.eventTime = eventtime;
- this.componentID = componentID;
- this.fileOffset = fileOffset;
+
+ // For testing only.. invoked via reflection
+ private void forceCloseLockFile() throws IOException {
+ lockFileStream.close();
}
- public String toString() {
- return eventTime + "," + componentID + "," + fileOffset;
+ public LogEntry getLastLogEntry() {
+ return lastEntry;
}
- public static LogEntry deserialize(String line) {
- String[] fields = line.split(",", NUM_FIELDS);
- return new LogEntry(Long.parseLong(fields[0]), fields[1], fields[2]);
+
+ public Path getLockFile() {
+ return lockFile;
}
- @Override
- public boolean equals(Object o) {
- if (this == o) { return true; }
- if (!(o instanceof LogEntry)) { return false; }
+ public static class LogEntry {
+ private static final int NUM_FIELDS = 3;
+ public final long eventTime;
+ public final String componentID;
+ public final String fileOffset;
- LogEntry logEntry = (LogEntry) o;
+ public LogEntry(long eventtime, String componentID, String fileOffset) {
+ this.eventTime = eventtime;
+ this.componentID = componentID;
+ this.fileOffset = fileOffset;
+ }
- if (eventTime != logEntry.eventTime) { return false; }
- if (!componentID.equals(logEntry.componentID)) { return false; }
- return fileOffset.equals(logEntry.fileOffset);
+ public static LogEntry deserialize(String line) {
+ String[] fields = line.split(",", NUM_FIELDS);
+ return new LogEntry(Long.parseLong(fields[0]), fields[1], fields[2]);
+ }
- }
+ public String toString() {
+ return eventTime + "," + componentID + "," + fileOffset;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof LogEntry)) {
+ return false;
+ }
+
+ LogEntry logEntry = (LogEntry) o;
+
+ if (eventTime != logEntry.eventTime) {
+ return false;
+ }
+ if (!componentID.equals(logEntry.componentID)) {
+ return false;
+ }
+ return fileOffset.equals(logEntry.fileOffset);
- @Override
- public int hashCode() {
- int result = (int) (eventTime ^ (eventTime >>> 32));
- result = 31 * result + componentID.hashCode();
- result = 31 * result + fileOffset.hashCode();
- return result;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = (int) (eventTime ^ (eventTime >>> 32));
+ result = 31 * result + componentID.hashCode();
+ result = 31 * result + fileOffset.hashCode();
+ return result;
+ }
}
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java
index 78296b9..bf58815 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileOffset.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
@@ -30,7 +24,8 @@ package org.apache.storm.hdfs.spout;
*/
interface FileOffset extends Comparable<FileOffset>, Cloneable {
- /** tests if rhs == currOffset+1 */
- boolean isNextOffset(FileOffset rhs);
- FileOffset clone();
+ /** tests if rhs == currOffset+1 */
+ boolean isNextOffset(FileOffset rhs);
+
+ FileOffset clone();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java
index 54a90d4..49d998a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/FileReader.java
@@ -1,44 +1,37 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
-import org.apache.hadoop.fs.Path;
-
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.fs.Path;
interface FileReader {
- Path getFilePath();
+ Path getFilePath();
- /**
- * A simple numeric value may not be sufficient for certain formats consequently
- * this is a String.
- */
- FileOffset getFileOffset();
+ /**
+ * A simple numeric value may not be sufficient for certain formats consequently
+ * this is a String.
+ */
+ FileOffset getFileOffset();
- /**
- * Get the next tuple from the file
- *
- * @return null if no more data
- * @throws IOException
- */
- List<Object> next() throws IOException, ParseException;
+ /**
+ * Get the next tuple from the file
+ *
+ * @return null if no more data
+ * @throws IOException
+ */
+ List<Object> next() throws IOException, ParseException;
- void close();
+ void close();
}
[34/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-cassandra
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-cassandra
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1a2d131f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1a2d131f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1a2d131f
Branch: refs/heads/master
Commit: 1a2d131f99e62823bf15fa958cc676a51efda10c
Parents: e3f5b13
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 22:44:56 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 00:22:36 2018 -0400
----------------------------------------------------------------------
external/storm-cassandra/pom.xml | 2 +-
.../AbstractExecutionResultHandler.java | 34 ++--
.../cassandra/BaseExecutionResultHandler.java | 32 ++--
.../storm/cassandra/CassandraContext.java | 28 ++-
.../cassandra/DynamicStatementBuilder.java | 38 ++--
.../storm/cassandra/ExecutionResultHandler.java | 27 ++-
.../storm/cassandra/Murmur3StreamGrouping.java | 87 ++++-----
.../storm/cassandra/bolt/BaseCassandraBolt.java | 42 ++--
.../bolt/BatchCassandraWriterBolt.java | 80 ++++----
.../cassandra/bolt/CassandraWriterBolt.java | 34 ++--
.../cassandra/bolt/GroupingBatchBuilder.java | 29 ++-
.../bolt/PairBatchStatementTuples.java | 23 +--
.../cassandra/bolt/PairStatementTuple.java | 26 +--
.../storm/cassandra/client/CassandraConf.java | 191 +++++++++----------
.../storm/cassandra/client/ClusterFactory.java | 46 ++---
.../storm/cassandra/client/SimpleClient.java | 20 +-
.../cassandra/client/SimpleClientProvider.java | 20 +-
.../cassandra/client/impl/DefaultClient.java | 27 +--
.../cassandra/context/BaseBeanFactory.java | 25 ++-
.../storm/cassandra/context/BeanFactory.java | 20 +-
.../storm/cassandra/context/WorkerCtx.java | 32 ++--
.../storm/cassandra/executor/AsyncExecutor.java | 98 +++++-----
.../executor/AsyncExecutorProvider.java | 22 +--
.../cassandra/executor/AsyncResultHandler.java | 27 +--
.../executor/AsyncResultSetHandler.java | 23 +--
.../executor/ExecutionResultCollector.java | 36 ++--
.../executor/impl/BatchAsyncResultHandler.java | 32 ++--
.../executor/impl/SingleAsyncResultHandler.java | 30 ++-
.../query/AyncCQLResultSetValuesMapper.java | 25 +--
.../query/BaseCQLStatementTupleMapper.java | 23 +--
.../query/CQLResultSetValuesMapper.java | 25 +--
.../cassandra/query/CQLStatementBuilder.java | 20 +-
.../query/CQLStatementTupleMapper.java | 25 +--
.../apache/storm/cassandra/query/Column.java | 34 ++--
.../storm/cassandra/query/ContextQuery.java | 23 +--
.../apache/storm/cassandra/query/CqlMapper.java | 29 ++-
.../cassandra/query/ObjectMapperOperation.java | 19 +-
.../builder/BoundCQLStatementMapperBuilder.java | 33 ++--
.../ObjectMapperCqlStatementMapperBuilder.java | 19 +-
.../SimpleCQLStatementMapperBuilder.java | 31 ++-
.../impl/BatchCQLStatementTupleMapper.java | 29 ++-
.../impl/BoundCQLStatementTupleMapper.java | 48 ++---
.../impl/ObjectMapperCqlStatementMapper.java | 27 ++-
.../query/impl/PreparedStatementBinder.java | 40 ++--
.../query/impl/RoutingKeyGenerator.java | 25 +--
.../query/impl/SimpleCQLStatementMapper.java | 36 ++--
.../cassandra/query/selector/FieldSelector.java | 25 +--
.../trident/state/CassandraBackingMap.java | 49 ++---
.../trident/state/CassandraMapStateFactory.java | 29 ++-
.../cassandra/trident/state/CassandraQuery.java | 25 +--
.../cassandra/trident/state/CassandraState.java | 91 ++++-----
.../trident/state/CassandraStateFactory.java | 28 ++-
.../trident/state/CassandraStateUpdater.java | 23 +--
.../trident/state/MapStateFactoryBuilder.java | 85 ++++-----
.../state/NonTransactionalTupleStateMapper.java | 23 +--
.../trident/state/OpaqueTupleStateMapper.java | 31 ++-
.../trident/state/SerializedStateMapper.java | 31 ++-
.../trident/state/SimpleStateMapper.java | 26 +--
.../cassandra/trident/state/SimpleTuple.java | 38 ++--
.../cassandra/trident/state/StateMapper.java | 25 +--
.../state/TransactionalTupleStateMapper.java | 28 +--
.../TridentAyncCQLResultSetValuesMapper.java | 28 +--
.../state/TridentResultSetValuesMapper.java | 29 ++-
.../apache/storm/cassandra/WeatherSpout.java | 25 +--
.../testtools/EmbeddedCassandraResource.java | 33 ++--
.../storm/cassandra/trident/MapStateTest.java | 103 +++++-----
.../cassandra/trident/WeatherBatchSpout.java | 43 ++---
67 files changed, 1011 insertions(+), 1449 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/pom.xml b/external/storm-cassandra/pom.xml
index d98824c..87471c8 100644
--- a/external/storm-cassandra/pom.xml
+++ b/external/storm-cassandra/pom.xml
@@ -132,7 +132,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>578</maxAllowedViolations>
+ <maxAllowedViolations>159</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/AbstractExecutionResultHandler.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/AbstractExecutionResultHandler.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/AbstractExecutionResultHandler.java
index 6510565..3b32749 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/AbstractExecutionResultHandler.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/AbstractExecutionResultHandler.java
@@ -1,35 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.cassandra;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Tuple;
import com.datastax.driver.core.exceptions.QueryValidationException;
import com.datastax.driver.core.exceptions.ReadTimeoutException;
import com.datastax.driver.core.exceptions.UnavailableException;
import com.datastax.driver.core.exceptions.WriteTimeoutException;
+import java.util.List;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.List;
-
/**
* Default interface to define strategies to apply when a query is either succeed or failed.
*
@@ -40,10 +32,10 @@ public abstract class AbstractExecutionResultHandler implements ExecutionResultH
@Override
public void onThrowable(Throwable t, OutputCollector collector, Tuple i) {
- if( t instanceof QueryValidationException) {
- this.onQueryValidationException((QueryValidationException)t, collector, i);
+ if (t instanceof QueryValidationException) {
+ this.onQueryValidationException((QueryValidationException) t, collector, i);
} else if (t instanceof ReadTimeoutException) {
- this.onReadTimeoutException((ReadTimeoutException)t, collector, i);
+ this.onReadTimeoutException((ReadTimeoutException) t, collector, i);
} else if (t instanceof WriteTimeoutException) {
this.onWriteTimeoutException((WriteTimeoutException) t, collector, i);
} else if (t instanceof UnavailableException) {
@@ -56,6 +48,6 @@ public abstract class AbstractExecutionResultHandler implements ExecutionResultH
@Override
public void onThrowable(Throwable t, OutputCollector collector, List<Tuple> tl) {
- for(Tuple i : tl) onThrowable(t, collector, i);
+ for (Tuple i : tl) onThrowable(t, collector, i);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/BaseExecutionResultHandler.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/BaseExecutionResultHandler.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/BaseExecutionResultHandler.java
index 59b6343..57ef351 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/BaseExecutionResultHandler.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/BaseExecutionResultHandler.java
@@ -1,31 +1,29 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.cassandra;
+import com.datastax.driver.core.exceptions.DriverException;
+import com.datastax.driver.core.exceptions.QueryValidationException;
+import com.datastax.driver.core.exceptions.ReadTimeoutException;
+import com.datastax.driver.core.exceptions.UnavailableException;
+import com.datastax.driver.core.exceptions.WriteTimeoutException;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
-import com.datastax.driver.core.exceptions.*;
import org.slf4j.LoggerFactory;
/**
- * Simple {@link ExecutionResultHandler} which fail the incoming tuple when an {@link com.datastax.driver.core.exceptions.DriverException} is thrown.
+ * Simple {@link ExecutionResultHandler} which fail the incoming tuple when an
+ * {@link com.datastax.driver.core.exceptions.DriverException} is thrown.
* The exception is then automatically report to storm.
*
*/
@@ -40,6 +38,7 @@ public class BaseExecutionResultHandler extends AbstractExecutionResultHandler {
public void onQueryValidationException(QueryValidationException e, OutputCollector collector, Tuple tuple) {
onDriverException(e, collector, tuple);
}
+
/**
* {@inheritDoc}
*/
@@ -47,6 +46,7 @@ public class BaseExecutionResultHandler extends AbstractExecutionResultHandler {
public void onReadTimeoutException(ReadTimeoutException e, OutputCollector collector, Tuple tuple) {
onDriverException(e, collector, tuple);
}
+
/**
* {@inheritDoc}
*/
@@ -54,6 +54,7 @@ public class BaseExecutionResultHandler extends AbstractExecutionResultHandler {
public void onWriteTimeoutException(WriteTimeoutException e, OutputCollector collector, Tuple tuple) {
onDriverException(e, collector, tuple);
}
+
/**
* {@inheritDoc}
*/
@@ -61,6 +62,7 @@ public class BaseExecutionResultHandler extends AbstractExecutionResultHandler {
public void onUnavailableException(UnavailableException e, OutputCollector collector, Tuple tuple) {
onDriverException(e, collector, tuple);
}
+
/**
* {@inheritDoc}
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/CassandraContext.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/CassandraContext.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/CassandraContext.java
index e862755..49763d2 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/CassandraContext.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/CassandraContext.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.cassandra;
import com.datastax.driver.core.Cluster;
+import java.util.Map;
import org.apache.storm.cassandra.client.CassandraConf;
import org.apache.storm.cassandra.client.ClusterFactory;
import org.apache.storm.cassandra.client.SimpleClient;
@@ -30,8 +24,6 @@ import org.apache.storm.cassandra.context.WorkerCtx;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
-
/**
*
*/
@@ -52,8 +44,9 @@ public class CassandraContext extends WorkerCtx implements SimpleClientProvider
@Override
public SimpleClient getClient(Map<String, Object> config) {
SimpleClient client = getWorkerBean(SimpleClient.class, config);
- if (client.isClose() )
+ if (client.isClose()) {
client = getWorkerBean(SimpleClient.class, config, true);
+ }
return client;
}
@@ -76,13 +69,14 @@ public class CassandraContext extends WorkerCtx implements SimpleClientProvider
public static final class ClientFactory extends BaseBeanFactory<SimpleClient> {
private static final Logger LOG = LoggerFactory.getLogger(ClientFactory.class);
+
/**
* {@inheritDoc}
*/
@Override
protected SimpleClient make(Map<String, Object> topoConf) {
Cluster cluster = this.context.getWorkerBean(Cluster.class, topoConf);
- if( cluster.isClosed() ) {
+ if (cluster.isClosed()) {
LOG.warn("Cluster is closed - trigger new initialization!");
cluster = this.context.getWorkerBean(Cluster.class, topoConf, true);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/DynamicStatementBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/DynamicStatementBuilder.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/DynamicStatementBuilder.java
index 2bd7107..823a566 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/DynamicStatementBuilder.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/DynamicStatementBuilder.java
@@ -1,40 +1,32 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.cassandra;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.querybuilder.BuiltStatement;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import org.apache.storm.cassandra.query.CQLStatementBuilder;
import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
import org.apache.storm.cassandra.query.ContextQuery;
import org.apache.storm.cassandra.query.CqlMapper;
-import org.apache.storm.cassandra.query.impl.BatchCQLStatementTupleMapper;
import org.apache.storm.cassandra.query.builder.BoundCQLStatementMapperBuilder;
import org.apache.storm.cassandra.query.builder.SimpleCQLStatementMapperBuilder;
+import org.apache.storm.cassandra.query.impl.BatchCQLStatementTupleMapper;
import org.apache.storm.cassandra.query.selector.FieldSelector;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
public class DynamicStatementBuilder implements Serializable {
private DynamicStatementBuilder() {
@@ -84,12 +76,14 @@ public class DynamicStatementBuilder implements Serializable {
public static final BatchCQLStatementTupleMapper loggedBatch(CQLStatementBuilder... builders) {
return newBatchStatementBuilder(BatchStatement.Type.LOGGED, builders);
}
+
/**
* Creates a new {@link com.datastax.driver.core.BatchStatement.Type#COUNTER} batch statement for the specified CQL statement builders.
*/
public static final BatchCQLStatementTupleMapper counterBatch(CQLStatementBuilder... builders) {
return newBatchStatementBuilder(BatchStatement.Type.COUNTER, builders);
}
+
/**
* Creates a new {@link com.datastax.driver.core.BatchStatement.Type#UNLOGGED} batch statement for the specified CQL statement builders.
*/
@@ -99,7 +93,7 @@ public class DynamicStatementBuilder implements Serializable {
private static BatchCQLStatementTupleMapper newBatchStatementBuilder(BatchStatement.Type type, CQLStatementBuilder[] builders) {
List<CQLStatementTupleMapper> mappers = new ArrayList<>(builders.length);
- for(CQLStatementBuilder b : Arrays.asList(builders))
+ for (CQLStatementBuilder b : Arrays.asList(builders))
mappers.add(b.build());
return new BatchCQLStatementTupleMapper(type, mappers);
}
@@ -142,8 +136,8 @@ public class DynamicStatementBuilder implements Serializable {
public static final FieldSelector[] fields(final String... fields) {
int size = fields.length;
List<FieldSelector> fl = new ArrayList<>(size);
- for(int i = 0 ; i < size; i++)
- fl.add(new FieldSelector(fields[i]));
+ for (int i = 0; i < size; i++)
+ fl.add(new FieldSelector(fields[i]));
return fl.toArray(new FieldSelector[size]);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/ExecutionResultHandler.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/ExecutionResultHandler.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/ExecutionResultHandler.java
index 6113efb..0e2a4db 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/ExecutionResultHandler.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/ExecutionResultHandler.java
@@ -1,33 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.cassandra;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Tuple;
import com.datastax.driver.core.exceptions.QueryValidationException;
import com.datastax.driver.core.exceptions.ReadTimeoutException;
import com.datastax.driver.core.exceptions.UnavailableException;
import com.datastax.driver.core.exceptions.WriteTimeoutException;
-
import java.io.Serializable;
import java.util.List;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.tuple.Tuple;
/**
* Default interface to define strategies to apply when a query is either succeed or failed.
@@ -43,6 +35,7 @@ public interface ExecutionResultHandler extends Serializable {
* @param tuple an input tuple.
*/
void onQueryValidationException(QueryValidationException e, OutputCollector collector, Tuple tuple);
+
/**
* Invoked when a {@link com.datastax.driver.core.exceptions.ReadTimeoutException} is thrown.
*
@@ -60,6 +53,7 @@ public interface ExecutionResultHandler extends Serializable {
* @param tuple an input tuple.
*/
void onWriteTimeoutException(WriteTimeoutException e, OutputCollector collector, Tuple tuple);
+
/**
* Invoked when a {@link com.datastax.driver.core.exceptions.UnavailableException} is thrown.
*
@@ -69,6 +63,7 @@ public interface ExecutionResultHandler extends Serializable {
* @param tuple an input tuple.
*/
void onUnavailableException(UnavailableException e, OutputCollector collector, Tuple tuple);
+
/**
* Invoked when a query is executed with success.
* This method is NOT responsible for acknowledging input tuple.
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/Murmur3StreamGrouping.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/Murmur3StreamGrouping.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/Murmur3StreamGrouping.java
index 774955a..c172cb6 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/Murmur3StreamGrouping.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/Murmur3StreamGrouping.java
@@ -1,39 +1,31 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.cassandra;
-import org.apache.storm.generated.GlobalStreamId;
-import org.apache.storm.grouping.CustomStreamGrouping;
-import org.apache.storm.task.WorkerTopologyContext;
-import org.apache.storm.topology.FailedException;
-import org.apache.storm.tuple.Fields;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.hash.Hashing;
-
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.storm.generated.GlobalStreamId;
+import org.apache.storm.grouping.CustomStreamGrouping;
+import org.apache.storm.task.WorkerTopologyContext;
+import org.apache.storm.topology.FailedException;
+import org.apache.storm.tuple.Fields;
/**
*
@@ -57,8 +49,8 @@ public class Murmur3StreamGrouping implements CustomStreamGrouping {
* Creates a new {@link Murmur3StreamGrouping} instance.
* @param partitionKeyNames {@link org.apache.storm.cassandra.Murmur3StreamGrouping#partitionKeyNames}.
*/
- public Murmur3StreamGrouping(String...partitionKeyNames) {
- this( Arrays.asList(partitionKeyNames));
+ public Murmur3StreamGrouping(String... partitionKeyNames) {
+ this(Arrays.asList(partitionKeyNames));
}
/**
@@ -69,6 +61,30 @@ public class Murmur3StreamGrouping implements CustomStreamGrouping {
this.partitionKeyNames = partitionKeyNames;
}
+ /**
+ * Computes the murmur3 hash for the specified values.
+ * http://stackoverflow.com/questions/27212797/cassandra-hashing-algorithm-with-composite-keys
+ * https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/db/marshal/CompositeType.java
+ *
+ * @param values the fields which are part of the (compose) partition key.
+ * @return the computed hash for input values.
+ * @throws java.io.IOException
+ */
+ @VisibleForTesting
+ public static long hashes(List<Object> values) throws IOException {
+ byte[] keyBytes;
+ try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(bos)) {
+ for (Object key : values) {
+ byte[] arr = ((String) key).getBytes("UTF-8");
+ out.writeShort(arr.length);
+ out.write(arr, 0, arr.length);
+ out.writeByte(0);
+ }
+ out.flush();
+ keyBytes = bos.toByteArray();
+ }
+ return Hashing.murmur3_128().hashBytes(keyBytes).asLong();
+ }
/**
* {@inheritDoc}
@@ -90,7 +106,7 @@ public class Murmur3StreamGrouping implements CustomStreamGrouping {
@Override
public List<Integer> chooseTasks(int taskId, List<Object> values) {
try {
- int n = Math.abs( (int) hashes(getKeyValues(values)) % targetTasks.size() );
+ int n = Math.abs((int) hashes(getKeyValues(values)) % targetTasks.size());
return Lists.newArrayList(targetTasks.get(n));
} catch (IOException e) {
throw new FailedException(e);
@@ -99,34 +115,9 @@ public class Murmur3StreamGrouping implements CustomStreamGrouping {
private List<Object> getKeyValues(List<Object> values) {
List<Object> keys = new ArrayList<>();
- for(Integer idx : partitionKeyIndexes) {
+ for (Integer idx : partitionKeyIndexes) {
keys.add(values.get(idx));
}
return keys;
}
-
- /**
- * Computes the murmur3 hash for the specified values.
- * http://stackoverflow.com/questions/27212797/cassandra-hashing-algorithm-with-composite-keys
- * https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/db/marshal/CompositeType.java
- *
- * @param values the fields which are part of the (compose) partition key.
- * @return the computed hash for input values.
- * @throws java.io.IOException
- */
- @VisibleForTesting
- public static long hashes(List<Object> values) throws IOException {
- byte[] keyBytes;
- try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(bos)) {
- for(Object key : values) {
- byte[] arr = ((String)key).getBytes("UTF-8");
- out.writeShort(arr.length);
- out.write(arr, 0, arr.length);
- out.writeByte(0);
- }
- out.flush();
- keyBytes = bos.toByteArray();
- }
- return Hashing.murmur3_128().hashBytes(keyBytes).asLong();
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/BaseCassandraBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/BaseCassandraBolt.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/BaseCassandraBolt.java
index f02fa26..946932f 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/BaseCassandraBolt.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/BaseCassandraBolt.java
@@ -1,26 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.bolt;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.exceptions.NoHostAvailableException;
import com.google.common.base.Preconditions;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.cassandra.BaseExecutionResultHandler;
import org.apache.storm.cassandra.CassandraContext;
@@ -42,9 +38,6 @@ import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.Map;
-
/**
* A base cassandra bolt.
*
@@ -55,7 +48,7 @@ public abstract class BaseCassandraBolt<T> extends BaseTickTupleAwareRichBolt {
private static final Logger LOG = LoggerFactory.getLogger(BaseCassandraBolt.class);
protected OutputCollector outputCollector;
-
+
protected SimpleClientProvider clientProvider;
protected SimpleClient client;
protected Session session;
@@ -66,7 +59,7 @@ public abstract class BaseCassandraBolt<T> extends BaseTickTupleAwareRichBolt {
private CQLStatementTupleMapper mapper;
private ExecutionResultHandler resultHandler;
- transient private Map<String, Fields> outputsFields = new HashMap<>();
+ transient private Map<String, Fields> outputsFields = new HashMap<>();
private Map<String, Object> cassandraConfig;
/**
@@ -77,6 +70,7 @@ public abstract class BaseCassandraBolt<T> extends BaseTickTupleAwareRichBolt {
this.mapper = mapper;
this.clientProvider = clientProvider;
}
+
/**
* Creates a new {@link CassandraWriterBolt} instance.
* @param tupleMapper
@@ -129,7 +123,7 @@ public abstract class BaseCassandraBolt<T> extends BaseTickTupleAwareRichBolt {
* @param fields
*/
public BaseCassandraBolt withStreamOutputFields(String stream, Fields fields) {
- if( stream == null || stream.length() == 0) throw new IllegalArgumentException("'stream' should not be null");
+ if (stream == null || stream.length() == 0) throw new IllegalArgumentException("'stream' should not be null");
this.outputsFields.put(stream, fields);
return this;
}
@@ -141,7 +135,7 @@ public abstract class BaseCassandraBolt<T> extends BaseTickTupleAwareRichBolt {
* @param config
*/
public BaseCassandraBolt withCassandraConfig(Map<String, Object> config) {
- if(config == null) {
+ if (config == null) {
throw new IllegalArgumentException("config should not be null");
}
cassandraConfig = config;
@@ -149,7 +143,7 @@ public abstract class BaseCassandraBolt<T> extends BaseTickTupleAwareRichBolt {
}
protected ExecutionResultHandler getResultHandler() {
- if(resultHandler == null) resultHandler = new BaseExecutionResultHandler();
+ if (resultHandler == null) resultHandler = new BaseExecutionResultHandler();
return resultHandler;
}
@@ -157,7 +151,7 @@ public abstract class BaseCassandraBolt<T> extends BaseTickTupleAwareRichBolt {
return mapper;
}
- abstract protected AsyncResultHandler<T> getAsyncHandler() ;
+ abstract protected AsyncResultHandler<T> getAsyncHandler();
protected AsyncExecutor<T> getAsyncExecutor() {
return AsyncExecutorProvider.getLocal(session, getAsyncHandler());
@@ -182,8 +176,8 @@ public abstract class BaseCassandraBolt<T> extends BaseTickTupleAwareRichBolt {
// outputsFields can be empty if this bolt acts like a sink in topology.
if (!outputsFields.isEmpty()) {
Fields fields = outputsFields.remove(Utils.DEFAULT_STREAM_ID);
- if( fields != null) declarer.declare(fields);
- for(Map.Entry<String, Fields> entry : outputsFields.entrySet()) {
+ if (fields != null) declarer.declare(fields);
+ for (Map.Entry<String, Fields> entry : outputsFields.entrySet()) {
declarer.declareStream(entry.getKey(), entry.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/BatchCassandraWriterBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/BatchCassandraWriterBolt.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/BatchCassandraWriterBolt.java
index 292f36d..d87ecab 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/BatchCassandraWriterBolt.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/BatchCassandraWriterBolt.java
@@ -1,51 +1,42 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.bolt;
+import com.datastax.driver.core.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
import org.apache.storm.Config;
+import org.apache.storm.cassandra.executor.AsyncResultHandler;
+import org.apache.storm.cassandra.executor.impl.BatchAsyncResultHandler;
+import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.utils.Time;
-import com.datastax.driver.core.Statement;
-import org.apache.storm.cassandra.executor.AsyncResultHandler;
-import org.apache.storm.cassandra.executor.impl.BatchAsyncResultHandler;
-import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
public class BatchCassandraWriterBolt extends BaseCassandraBolt<List<Tuple>> {
- private final static Logger LOG = LoggerFactory.getLogger(BatchCassandraWriterBolt.class);
-
public static final int DEFAULT_EMIT_FREQUENCY = 2;
-
+ private final static Logger LOG = LoggerFactory.getLogger(BatchCassandraWriterBolt.class);
private LinkedBlockingQueue<Tuple> queue;
-
+
private int tickFrequencyInSeconds;
-
+
private long lastModifiedTimesMillis;
private int batchMaxSize = 1000;
@@ -72,6 +63,7 @@ public class BatchCassandraWriterBolt extends BaseCassandraBolt<List<Tuple>> {
super(tupleMapper);
this.tickFrequencyInSeconds = tickFrequencyInSeconds;
}
+
/**
* {@inheritDoc}
*/
@@ -85,7 +77,7 @@ public class BatchCassandraWriterBolt extends BaseCassandraBolt<List<Tuple>> {
@Override
protected AsyncResultHandler<List<Tuple>> getAsyncHandler() {
- if( asyncResultHandler == null) {
+ if (asyncResultHandler == null) {
asyncResultHandler = new BatchAsyncResultHandler(getResultHandler());
}
return asyncResultHandler;
@@ -96,12 +88,13 @@ public class BatchCassandraWriterBolt extends BaseCassandraBolt<List<Tuple>> {
*/
@Override
protected void process(Tuple input) {
- if( ! queue.offer(input) ) {
+ if (!queue.offer(input)) {
LOG.info(logPrefix() + "The message queue is full, preparing batch statement...");
prepareAndExecuteStatement();
queue.add(input);
}
}
+
/**
* {@inheritDoc}
*/
@@ -112,7 +105,7 @@ public class BatchCassandraWriterBolt extends BaseCassandraBolt<List<Tuple>> {
public void prepareAndExecuteStatement() {
int size = queue.size();
- if( size > 0 ) {
+ if (size > 0) {
List<Tuple> inputs = new ArrayList<>(size);
queue.drainTo(inputs);
try {
@@ -127,16 +120,18 @@ public class BatchCassandraWriterBolt extends BaseCassandraBolt<List<Tuple>> {
int batchSize = 0;
for (PairBatchStatementTuples batch : batchBuilder) {
- LOG.debug(logPrefix() + "Writing data to {} in batches of {} rows.", cassandraConf.getKeyspace(), batch.getInputs().size());
+ LOG.debug(logPrefix() + "Writing data to {} in batches of {} rows.", cassandraConf.getKeyspace(),
+ batch.getInputs().size());
getAsyncExecutor().execAsync(batch.getStatement(), batch.getInputs());
batchSize++;
}
int pending = getAsyncExecutor().getPendingTasksSize();
if (pending > batchSize) {
- LOG.warn( logPrefix() + "Currently pending tasks is superior to the number of submit batches({}) : {}", batchSize, pending);
+ LOG.warn(logPrefix() + "Currently pending tasks is superior to the number of submit batches({}) : {}", batchSize,
+ pending);
}
-
+
} catch (Throwable r) {
LOG.error(logPrefix() + "Error(s) occurred while preparing batch statements");
getAsyncHandler().failure(r, inputs);
@@ -147,17 +142,19 @@ public class BatchCassandraWriterBolt extends BaseCassandraBolt<List<Tuple>> {
private List<PairStatementTuple> buildStatement(List<Tuple> inputs) {
List<PairStatementTuple> stmts = new ArrayList<>(inputs.size());
- for(Tuple t : inputs) {
+ for (Tuple t : inputs) {
List<Statement> sl = getMapper().map(topoConfig, session, t);
- for(Statement s : sl)
- stmts.add(new PairStatementTuple(t, s) );
+ for (Statement s : sl)
+ stmts.add(new PairStatementTuple(t, s));
}
return stmts;
}
private void checkTimeElapsedSinceLastExec(int sinceLastModified) {
- if(sinceLastModified > tickFrequencyInSeconds)
- LOG.warn( logPrefix() + "The time elapsed since last execution exceeded tick tuple frequency - {} > {} seconds", sinceLastModified, tickFrequencyInSeconds);
+ if (sinceLastModified > tickFrequencyInSeconds) {
+ LOG.warn(logPrefix() + "The time elapsed since last execution exceeded tick tuple frequency - {} > {} seconds",
+ sinceLastModified, tickFrequencyInSeconds);
+ }
}
private String logPrefix() {
@@ -165,7 +162,7 @@ public class BatchCassandraWriterBolt extends BaseCassandraBolt<List<Tuple>> {
}
public BatchCassandraWriterBolt withTickFrequency(long time, TimeUnit unit) {
- this.tickFrequencyInSeconds = (int)unit.toSeconds(time);
+ this.tickFrequencyInSeconds = (int) unit.toSeconds(time);
return this;
}
@@ -178,6 +175,7 @@ public class BatchCassandraWriterBolt extends BaseCassandraBolt<List<Tuple>> {
this.batchMaxSize = size;
return this;
}
+
/**
* {@inheritDoc}
*/
@@ -187,7 +185,7 @@ public class BatchCassandraWriterBolt extends BaseCassandraBolt<List<Tuple>> {
conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, tickFrequencyInSeconds);
return conf;
}
-
+
private int updateAndGetSecondsSinceLastModified() {
long now = now();
int seconds = (int) (now - lastModifiedTimesMillis) / 1000;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/CassandraWriterBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/CassandraWriterBolt.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/CassandraWriterBolt.java
index 876f6f5..fc45b3f 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/CassandraWriterBolt.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/CassandraWriterBolt.java
@@ -1,30 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.bolt;
-import org.apache.storm.tuple.Tuple;
import com.datastax.driver.core.Statement;
+import java.util.List;
import org.apache.storm.cassandra.executor.AsyncResultHandler;
import org.apache.storm.cassandra.executor.impl.SingleAsyncResultHandler;
import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
-
-import java.util.List;
+import org.apache.storm.tuple.Tuple;
public class CassandraWriterBolt extends BaseCassandraBolt<Tuple> {
@@ -44,7 +37,7 @@ public class CassandraWriterBolt extends BaseCassandraBolt<Tuple> {
*/
@Override
protected AsyncResultHandler<Tuple> getAsyncHandler() {
- if( asyncResultHandler == null) {
+ if (asyncResultHandler == null) {
asyncResultHandler = new SingleAsyncResultHandler(getResultHandler());
}
return asyncResultHandler;
@@ -56,8 +49,11 @@ public class CassandraWriterBolt extends BaseCassandraBolt<Tuple> {
@Override
protected void process(Tuple input) {
List<Statement> statements = getMapper().map(topoConfig, session, input);
- if (statements.size() == 1) getAsyncExecutor().execAsync(statements.get(0), input);
- else getAsyncExecutor().execAsync(statements, input);
+ if (statements.size() == 1) {
+ getAsyncExecutor().execAsync(statements.get(0), input);
+ } else {
+ getAsyncExecutor().execAsync(statements, input);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/GroupingBatchBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/GroupingBatchBuilder.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/GroupingBatchBuilder.java
index fdafd50..3697f81 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/GroupingBatchBuilder.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/GroupingBatchBuilder.java
@@ -1,39 +1,32 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.bolt;
-import org.apache.storm.tuple.Tuple;
import com.datastax.driver.core.BatchStatement;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
-
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import org.apache.storm.tuple.Tuple;
/**
- *
+ *
*/
public class GroupingBatchBuilder implements Iterable<PairBatchStatementTuples> {
private int batchSizeRows;
-
+
private List<PairStatementTuple> statements;
/**
@@ -50,7 +43,7 @@ public class GroupingBatchBuilder implements Iterable<PairBatchStatementTuples>
return build().iterator();
}
- private Iterable<PairBatchStatementTuples> build( ) {
+ private Iterable<PairBatchStatementTuples> build() {
Iterable<List<PairStatementTuple>> partition = Iterables.partition(statements, batchSizeRows);
return Iterables.transform(partition, new Function<List<PairStatementTuple>, PairBatchStatementTuples>() {
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/PairBatchStatementTuples.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/PairBatchStatementTuples.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/PairBatchStatementTuples.java
index cef422e..e6eef06 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/PairBatchStatementTuples.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/PairBatchStatementTuples.java
@@ -1,27 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.bolt;
-import org.apache.storm.tuple.Tuple;
import com.datastax.driver.core.BatchStatement;
-
import java.util.List;
+import org.apache.storm.tuple.Tuple;
/**
* Simple class to pair a list of tuples with a single batch statement.
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/PairStatementTuple.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/PairStatementTuple.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/PairStatementTuple.java
index 0f501a3..d95e9d3 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/PairStatementTuple.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/bolt/PairStatementTuple.java
@@ -1,33 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.bolt;
-import org.apache.storm.tuple.Tuple;
import com.datastax.driver.core.Statement;
+import org.apache.storm.tuple.Tuple;
/**
* Simple class to pair a tuple with a statement.
*/
public class PairStatementTuple {
-
+
private final Tuple tuple;
-
+
private final Statement statement;
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/CassandraConf.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/CassandraConf.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/CassandraConf.java
index 19b7551..82f017f 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/CassandraConf.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/CassandraConf.java
@@ -1,68 +1,61 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.client;
+import com.datastax.driver.core.ConsistencyLevel;
import com.datastax.driver.core.SocketOptions;
import com.datastax.driver.core.policies.DCAwareRoundRobinPolicy;
import com.datastax.driver.core.policies.DCAwareRoundRobinPolicy.Builder;
-import com.datastax.driver.core.policies.LoadBalancingPolicy;
-import com.datastax.driver.core.policies.LoggingRetryPolicy;
-import com.datastax.driver.core.policies.RoundRobinPolicy;
-import com.datastax.driver.core.policies.TokenAwarePolicy;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.storm.utils.Utils;
-import org.apache.storm.utils.ObjectReader;
-import com.datastax.driver.core.ConsistencyLevel;
import com.datastax.driver.core.policies.DefaultRetryPolicy;
import com.datastax.driver.core.policies.DowngradingConsistencyRetryPolicy;
import com.datastax.driver.core.policies.FallthroughRetryPolicy;
+import com.datastax.driver.core.policies.LoadBalancingPolicy;
+import com.datastax.driver.core.policies.LoggingRetryPolicy;
import com.datastax.driver.core.policies.RetryPolicy;
+import com.datastax.driver.core.policies.RoundRobinPolicy;
+import com.datastax.driver.core.policies.TokenAwarePolicy;
import com.google.common.base.Objects;
-
import java.io.Serializable;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.Utils;
/**
* Configuration used by cassandra storm components.
*/
public class CassandraConf implements Serializable {
-
- public static final String CASSANDRA_USERNAME = "cassandra.username";
- public static final String CASSANDRA_PASSWORD = "cassandra.password";
- public static final String CASSANDRA_KEYSPACE = "cassandra.keyspace";
- public static final String CASSANDRA_CONSISTENCY_LEVEL = "cassandra.output.consistencyLevel";
- public static final String CASSANDRA_NODES = "cassandra.nodes";
- public static final String CASSANDRA_PORT = "cassandra.port";
- public static final String CASSANDRA_BATCH_SIZE_ROWS = "cassandra.batch.size.rows";
- public static final String CASSANDRA_RETRY_POLICY = "cassandra.retryPolicy";
- public static final String CASSANDRA_RECONNECT_POLICY_BASE_MS = "cassandra.reconnectionPolicy.baseDelayMs";
- public static final String CASSANDRA_RECONNECT_POLICY_MAX_MS = "cassandra.reconnectionPolicy.maxDelayMs";
- public static final String CASSANDRA_POOL_MAX_SIZE = "cassandra.pool.max.size";
- public static final String CASSANDRA_LOAD_BALANCING_POLICY = "cassandra.loadBalancingPolicy";
- public static final String CASSANDRA_DATACENTER_NAME = "cassandra.datacenter.name";
- public static final String CASSANDRA_MAX_REQUESTS_PER_CON_LOCAL = "cassandra.max.requests.per.con.local";
+
+ public static final String CASSANDRA_USERNAME = "cassandra.username";
+ public static final String CASSANDRA_PASSWORD = "cassandra.password";
+ public static final String CASSANDRA_KEYSPACE = "cassandra.keyspace";
+ public static final String CASSANDRA_CONSISTENCY_LEVEL = "cassandra.output.consistencyLevel";
+ public static final String CASSANDRA_NODES = "cassandra.nodes";
+ public static final String CASSANDRA_PORT = "cassandra.port";
+ public static final String CASSANDRA_BATCH_SIZE_ROWS = "cassandra.batch.size.rows";
+ public static final String CASSANDRA_RETRY_POLICY = "cassandra.retryPolicy";
+ public static final String CASSANDRA_RECONNECT_POLICY_BASE_MS = "cassandra.reconnectionPolicy.baseDelayMs";
+ public static final String CASSANDRA_RECONNECT_POLICY_MAX_MS = "cassandra.reconnectionPolicy.maxDelayMs";
+ public static final String CASSANDRA_POOL_MAX_SIZE = "cassandra.pool.max.size";
+ public static final String CASSANDRA_LOAD_BALANCING_POLICY = "cassandra.loadBalancingPolicy";
+ public static final String CASSANDRA_DATACENTER_NAME = "cassandra.datacenter.name";
+ public static final String CASSANDRA_MAX_REQUESTS_PER_CON_LOCAL = "cassandra.max.requests.per.con.local";
public static final String CASSANDRA_MAX_REQUESTS_PER_CON_REMOTE = "cassandra.max.requests.per.con.remote";
- public static final String CASSANDRA_HEARTBEAT_INTERVAL_SEC = "cassandra.heartbeat.interval.sec";
- public static final String CASSANDRA_IDLE_TIMEOUT_SEC = "cassandra.idle.timeout.sec";
- public static final String CASSANDRA_SOCKET_READ_TIMEOUT_MS = "cassandra.socket.read.timeout.millis";
- public static final String CASSANDRA_SOCKET_CONNECT_TIMEOUT_MS = "cassandra.socket.connect.timeout.millis";
+ public static final String CASSANDRA_HEARTBEAT_INTERVAL_SEC = "cassandra.heartbeat.interval.sec";
+ public static final String CASSANDRA_IDLE_TIMEOUT_SEC = "cassandra.idle.timeout.sec";
+ public static final String CASSANDRA_SOCKET_READ_TIMEOUT_MS = "cassandra.socket.read.timeout.millis";
+ public static final String CASSANDRA_SOCKET_CONNECT_TIMEOUT_MS = "cassandra.socket.connect.timeout.millis";
/**
* The authorized cassandra username.
@@ -79,7 +72,7 @@ public class CassandraConf implements Serializable {
/**
* List of contacts nodes.
*/
- private String[] nodes = {"localhost"};
+ private String[] nodes = { "localhost" };
/**
* The port used to connect to nodes.
@@ -93,7 +86,7 @@ public class CassandraConf implements Serializable {
/**
* The maximal numbers of rows per batch.
*/
- private int batchSizeRows = 100;
+ private int batchSizeRows = 100;
/**
* The retry policy to use for the new cluster.
@@ -154,22 +147,49 @@ public class CassandraConf implements Serializable {
this.username = (String) Utils.get(conf, CASSANDRA_USERNAME, null);
this.password = (String) Utils.get(conf, CASSANDRA_PASSWORD, null);
this.keyspace = get(conf, CASSANDRA_KEYSPACE);
- this.consistencyLevel = ConsistencyLevel.valueOf((String) Utils.get(conf, CASSANDRA_CONSISTENCY_LEVEL, ConsistencyLevel.ONE.name()));
- this.nodes = ((String) Utils.get(conf, CASSANDRA_NODES, "localhost")).split(",");
+ this.consistencyLevel =
+ ConsistencyLevel.valueOf((String) Utils.get(conf, CASSANDRA_CONSISTENCY_LEVEL, ConsistencyLevel.ONE.name()));
+ this.nodes = ((String) Utils.get(conf, CASSANDRA_NODES, "localhost")).split(",");
this.batchSizeRows = ObjectReader.getInt(conf.get(CASSANDRA_BATCH_SIZE_ROWS), 100);
this.port = ObjectReader.getInt(conf.get(CASSANDRA_PORT), 9042);
this.retryPolicyName = (String) Utils.get(conf, CASSANDRA_RETRY_POLICY, DefaultRetryPolicy.class.getSimpleName());
this.reconnectionPolicyBaseMs = getLong(conf.get(CASSANDRA_RECONNECT_POLICY_BASE_MS), 100L);
- this.reconnectionPolicyMaxMs = getLong(conf.get(CASSANDRA_RECONNECT_POLICY_MAX_MS), TimeUnit.MINUTES.toMillis(1));
+ this.reconnectionPolicyMaxMs = getLong(conf.get(CASSANDRA_RECONNECT_POLICY_MAX_MS), TimeUnit.MINUTES.toMillis(1));
this.poolMaxQueueSize = getInt(conf.get(CASSANDRA_POOL_MAX_SIZE), 256);
this.loadBalancingPolicyName = (String) Utils.get(conf, CASSANDRA_LOAD_BALANCING_POLICY, TokenAwarePolicy.class.getSimpleName());
- this.datacenterName = (String)Utils.get(conf, CASSANDRA_DATACENTER_NAME, null);
+ this.datacenterName = (String) Utils.get(conf, CASSANDRA_DATACENTER_NAME, null);
this.maxRequestPerConnectionLocal = getInt(conf.get(CASSANDRA_MAX_REQUESTS_PER_CON_LOCAL), 1024);
this.maxRequestPerConnectionRemote = getInt(conf.get(CASSANDRA_MAX_REQUESTS_PER_CON_REMOTE), 256);
this.heartbeatIntervalSeconds = getInt(conf.get(CASSANDRA_HEARTBEAT_INTERVAL_SEC), 30);
this.idleTimeoutSeconds = getInt(conf.get(CASSANDRA_IDLE_TIMEOUT_SEC), 60);
- this.socketReadTimeoutMillis = getLong(conf.get(CASSANDRA_SOCKET_READ_TIMEOUT_MS), (long)SocketOptions.DEFAULT_READ_TIMEOUT_MILLIS);
- this.socketConnectTimeoutMillis = getLong(conf.get(CASSANDRA_SOCKET_CONNECT_TIMEOUT_MS), (long)SocketOptions.DEFAULT_CONNECT_TIMEOUT_MILLIS);
+ this.socketReadTimeoutMillis =
+ getLong(conf.get(CASSANDRA_SOCKET_READ_TIMEOUT_MS), (long) SocketOptions.DEFAULT_READ_TIMEOUT_MILLIS);
+ this.socketConnectTimeoutMillis =
+ getLong(conf.get(CASSANDRA_SOCKET_CONNECT_TIMEOUT_MS), (long) SocketOptions.DEFAULT_CONNECT_TIMEOUT_MILLIS);
+ }
+
+ public static Integer getInt(Object o, Integer defaultValue) {
+ if (null == o) {
+ return defaultValue;
+ }
+ if (o instanceof Number) {
+ return ((Number) o).intValue();
+ } else if (o instanceof String) {
+ return Integer.parseInt((String) o);
+ }
+ throw new IllegalArgumentException("Don't know how to convert " + o + " to int");
+ }
+
+ public static Long getLong(Object o, Long defaultValue) {
+ if (null == o) {
+ return defaultValue;
+ }
+ if (o instanceof Number) {
+ return ((Number) o).longValue();
+ } else if (o instanceof String) {
+ return Long.parseLong((String) o);
+ }
+ throw new IllegalArgumentException("Don't know how to convert " + o + " to long");
}
public String getUsername() {
@@ -209,12 +229,15 @@ public class CassandraConf implements Serializable {
}
public RetryPolicy getRetryPolicy() {
- if(this.retryPolicyName.equals(DowngradingConsistencyRetryPolicy.class.getSimpleName()))
+ if (this.retryPolicyName.equals(DowngradingConsistencyRetryPolicy.class.getSimpleName())) {
return new LoggingRetryPolicy(DowngradingConsistencyRetryPolicy.INSTANCE);
- if(this.retryPolicyName.equals(FallthroughRetryPolicy.class.getSimpleName()))
+ }
+ if (this.retryPolicyName.equals(FallthroughRetryPolicy.class.getSimpleName())) {
return FallthroughRetryPolicy.INSTANCE;
- if(this.retryPolicyName.equals(DefaultRetryPolicy.class.getSimpleName()))
+ }
+ if (this.retryPolicyName.equals(DefaultRetryPolicy.class.getSimpleName())) {
return DefaultRetryPolicy.INSTANCE;
+ }
throw new IllegalArgumentException("Unknown cassandra retry policy " + this.retryPolicyName);
}
@@ -266,56 +289,32 @@ public class CassandraConf implements Serializable {
private <T> T get(Map<String, Object> conf, String key) {
Object o = conf.get(key);
- if(o == null) {
+ if (o == null) {
throw new IllegalArgumentException("No '" + key + "' value found in configuration!");
}
- return (T)o;
- }
-
- public static Integer getInt(Object o, Integer defaultValue) {
- if (null == o) {
- return defaultValue;
- }
- if (o instanceof Number) {
- return ((Number) o).intValue();
- } else if (o instanceof String) {
- return Integer.parseInt((String) o);
- }
- throw new IllegalArgumentException("Don't know how to convert " + o + " to int");
- }
-
- public static Long getLong(Object o, Long defaultValue) {
- if (null == o) {
- return defaultValue;
- }
- if (o instanceof Number) {
- return ((Number) o).longValue();
- } else if (o instanceof String) {
- return Long.parseLong((String) o);
- }
- throw new IllegalArgumentException("Don't know how to convert " + o + " to long");
+ return (T) o;
}
@Override
public String toString() {
return Objects.toStringHelper(this)
- .add("username", username)
- .add("password", password)
- .add("keyspace", keyspace)
- .add("nodes", nodes)
- .add("port", port)
- .add("consistencyLevel", consistencyLevel)
- .add("batchSizeRows", batchSizeRows)
- .add("retryPolicyName", retryPolicyName)
- .add("reconnectionPolicyBaseMs", reconnectionPolicyBaseMs)
- .add("reconnectionPolicyMaxMs", reconnectionPolicyMaxMs)
- .add("poolMaxQueueSize", poolMaxQueueSize)
- .add("datacenterName", datacenterName)
- .add("maxRequestPerConnectionLocal", maxRequestPerConnectionLocal)
- .add("maxRequestPerConnectionRemote", maxRequestPerConnectionRemote)
- .add("heartbeatIntervalSeconds", heartbeatIntervalSeconds)
- .add("idleTimeoutSeconds", idleTimeoutSeconds)
- .add("socketReadTimeoutMillis", socketReadTimeoutMillis)
- .toString();
+ .add("username", username)
+ .add("password", password)
+ .add("keyspace", keyspace)
+ .add("nodes", nodes)
+ .add("port", port)
+ .add("consistencyLevel", consistencyLevel)
+ .add("batchSizeRows", batchSizeRows)
+ .add("retryPolicyName", retryPolicyName)
+ .add("reconnectionPolicyBaseMs", reconnectionPolicyBaseMs)
+ .add("reconnectionPolicyMaxMs", reconnectionPolicyMaxMs)
+ .add("poolMaxQueueSize", poolMaxQueueSize)
+ .add("datacenterName", datacenterName)
+ .add("maxRequestPerConnectionLocal", maxRequestPerConnectionLocal)
+ .add("maxRequestPerConnectionRemote", maxRequestPerConnectionRemote)
+ .add("heartbeatIntervalSeconds", heartbeatIntervalSeconds)
+ .add("idleTimeoutSeconds", idleTimeoutSeconds)
+ .add("socketReadTimeoutMillis", socketReadTimeoutMillis)
+ .toString();
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/ClusterFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/ClusterFactory.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/ClusterFactory.java
index 407d29c..cfbe5a5 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/ClusterFactory.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/ClusterFactory.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.client;
import com.datastax.driver.core.Cluster;
@@ -43,27 +37,27 @@ public class ClusterFactory extends BaseBeanFactory<Cluster> {
CassandraConf cassandraConf = new CassandraConf(topoConf);
Cluster.Builder cluster = Cluster.builder()
- .withoutJMXReporting()
- .withoutMetrics()
- .addContactPoints(cassandraConf.getNodes())
- .withPort(cassandraConf.getPort())
- .withRetryPolicy(cassandraConf.getRetryPolicy())
- .withReconnectionPolicy(new ExponentialReconnectionPolicy(
- cassandraConf.getReconnectionPolicyBaseMs(),
- cassandraConf.getReconnectionPolicyMaxMs()))
- .withLoadBalancingPolicy(cassandraConf.getLoadBalancingPolicy());
- cluster.getConfiguration().getSocketOptions().setReadTimeoutMillis((int)cassandraConf.getSocketReadTimeoutMillis());
- cluster.getConfiguration().getSocketOptions().setConnectTimeoutMillis((int)cassandraConf.getSocketConnectTimeoutMillis());
+ .withoutJMXReporting()
+ .withoutMetrics()
+ .addContactPoints(cassandraConf.getNodes())
+ .withPort(cassandraConf.getPort())
+ .withRetryPolicy(cassandraConf.getRetryPolicy())
+ .withReconnectionPolicy(new ExponentialReconnectionPolicy(
+ cassandraConf.getReconnectionPolicyBaseMs(),
+ cassandraConf.getReconnectionPolicyMaxMs()))
+ .withLoadBalancingPolicy(cassandraConf.getLoadBalancingPolicy());
+ cluster.getConfiguration().getSocketOptions().setReadTimeoutMillis((int) cassandraConf.getSocketReadTimeoutMillis());
+ cluster.getConfiguration().getSocketOptions().setConnectTimeoutMillis((int) cassandraConf.getSocketConnectTimeoutMillis());
final String username = cassandraConf.getUsername();
final String password = cassandraConf.getPassword();
- if( StringUtils.isNotEmpty(username) && StringUtils.isNotEmpty(password)) {
+ if (StringUtils.isNotEmpty(username) && StringUtils.isNotEmpty(password)) {
cluster.withAuthProvider(new PlainTextAuthProvider(username, password));
}
QueryOptions options = new QueryOptions()
- .setConsistencyLevel(cassandraConf.getConsistencyLevel());
+ .setConsistencyLevel(cassandraConf.getConsistencyLevel());
cluster.withQueryOptions(options);
PoolingOptions poolOps = new PoolingOptions();
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/SimpleClient.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/SimpleClient.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/SimpleClient.java
index 3bd80ed..7df59dd 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/SimpleClient.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/SimpleClient.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.client;
import com.datastax.driver.core.Session;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/SimpleClientProvider.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/SimpleClientProvider.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/SimpleClientProvider.java
index 412cb70..588a26a 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/SimpleClientProvider.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/SimpleClientProvider.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.client;
import java.util.Map;
[59/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-hdfs-examples
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-hdfs-examples
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e8ffac45
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e8ffac45
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e8ffac45
Branch: refs/heads/master
Commit: e8ffac458baf6cc7a90dd0ced78cc01f0ffb4229
Parents: f7f3524
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Mon Apr 23 00:44:09 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 02:32:42 2018 -0400
----------------------------------------------------------------------
examples/storm-hdfs-examples/pom.xml | 2 +-
.../storm/hdfs/bolt/HdfsFileTopology.java | 54 ++--
.../storm/hdfs/bolt/SequenceFileTopology.java | 58 ++--
.../storm/hdfs/spout/HdfsSpoutTopology.java | 307 +++++++++----------
.../storm/hdfs/trident/FixedBatchSpout.java | 29 +-
.../storm/hdfs/trident/TridentFileTopology.java | 30 +-
.../hdfs/trident/TridentSequenceTopology.java | 30 +-
7 files changed, 248 insertions(+), 262 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e8ffac45/examples/storm-hdfs-examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/pom.xml b/examples/storm-hdfs-examples/pom.xml
index c2271fe..30b07e6 100644
--- a/examples/storm-hdfs-examples/pom.xml
+++ b/examples/storm-hdfs-examples/pom.xml
@@ -89,7 +89,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>224</maxAllowedViolations>
+ <maxAllowedViolations>29</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/e8ffac45/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
index f8d69ca..e3599d5 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt;
import java.io.FileInputStream;
@@ -23,7 +18,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat;
@@ -65,12 +59,12 @@ public class HdfsFileTopology {
FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);
FileNameFormat fileNameFormat = new DefaultFileNameFormat()
- .withPath("/tmp/foo/")
- .withExtension(".txt");
+ .withPath("/tmp/foo/")
+ .withExtension(".txt");
// use "|" instead of "," for field delimiter
RecordFormat format = new DelimitedRecordFormat()
- .withFieldDelimiter("|");
+ .withFieldDelimiter("|");
Yaml yaml = new Yaml();
InputStream in = new FileInputStream(args[1]);
@@ -79,20 +73,20 @@ public class HdfsFileTopology {
config.put("hdfs.config", yamlConf);
HdfsBolt bolt = new HdfsBolt()
- .withConfigKey("hdfs.config")
- .withFsUrl(args[0])
- .withFileNameFormat(fileNameFormat)
- .withRecordFormat(format)
- .withRotationPolicy(rotationPolicy)
- .withSyncPolicy(syncPolicy)
- .addRotationAction(new MoveFileAction().toDestination("/tmp/dest2/"));
+ .withConfigKey("hdfs.config")
+ .withFsUrl(args[0])
+ .withFileNameFormat(fileNameFormat)
+ .withRecordFormat(format)
+ .withRotationPolicy(rotationPolicy)
+ .withSyncPolicy(syncPolicy)
+ .addRotationAction(new MoveFileAction().toDestination("/tmp/dest2/"));
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SENTENCE_SPOUT_ID, spout, 1);
// SentenceSpout --> MyBolt
builder.setBolt(BOLT_ID, bolt, 4)
- .shuffleGrouping(SENTENCE_SPOUT_ID);
+ .shuffleGrouping(SENTENCE_SPOUT_ID);
String topoName = TOPOLOGY_NAME;
if (args.length == 3) {
topoName = args[2];
@@ -114,11 +108,11 @@ public class HdfsFileTopology {
private ConcurrentHashMap<UUID, Values> pending;
private SpoutOutputCollector collector;
private String[] sentences = {
- "my dog has fleas",
- "i like cold beverages",
- "the dog ate my homework",
- "don't have a cow man",
- "i don't think i like fleas"
+ "my dog has fleas",
+ "i like cold beverages",
+ "the dog ate my homework",
+ "don't have a cow man",
+ "i don't think i like fleas"
};
private int index = 0;
private int count = 0;
@@ -145,7 +139,7 @@ public class HdfsFileTopology {
}
count++;
total++;
- if(count > 20000){
+ if (count > 20000) {
count = 0;
System.out.println("Pending count: " + this.pending.size() + ", total: " + this.total);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8ffac45/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java
index 2707af5..8ddd045 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileTopology.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt;
import java.io.FileInputStream;
@@ -23,7 +18,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
-
import org.apache.hadoop.io.SequenceFile;
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
@@ -66,8 +60,8 @@ public class SequenceFileTopology {
FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB);
FileNameFormat fileNameFormat = new DefaultFileNameFormat()
- .withPath("/tmp/source/")
- .withExtension(".seq");
+ .withPath("/tmp/source/")
+ .withExtension(".seq");
// create sequence format instance.
DefaultSequenceFormat format = new DefaultSequenceFormat("timestamp", "sentence");
@@ -79,22 +73,22 @@ public class SequenceFileTopology {
config.put("hdfs.config", yamlConf);
SequenceFileBolt bolt = new SequenceFileBolt()
- .withFsUrl(args[0])
- .withConfigKey("hdfs.config")
- .withFileNameFormat(fileNameFormat)
- .withSequenceFormat(format)
- .withRotationPolicy(rotationPolicy)
- .withSyncPolicy(syncPolicy)
- .withCompressionType(SequenceFile.CompressionType.RECORD)
- .withCompressionCodec("deflate")
- .addRotationAction(new MoveFileAction().toDestination("/tmp/dest/"));
+ .withFsUrl(args[0])
+ .withConfigKey("hdfs.config")
+ .withFileNameFormat(fileNameFormat)
+ .withSequenceFormat(format)
+ .withRotationPolicy(rotationPolicy)
+ .withSyncPolicy(syncPolicy)
+ .withCompressionType(SequenceFile.CompressionType.RECORD)
+ .withCompressionCodec("deflate")
+ .addRotationAction(new MoveFileAction().toDestination("/tmp/dest/"));
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SENTENCE_SPOUT_ID, spout, 1);
// SentenceSpout --> MyBolt
builder.setBolt(BOLT_ID, bolt, 4)
- .shuffleGrouping(SENTENCE_SPOUT_ID);
+ .shuffleGrouping(SENTENCE_SPOUT_ID);
String topoName = TOPOLOGY_NAME;
if (args.length == 3) {
@@ -120,11 +114,11 @@ public class SequenceFileTopology {
private ConcurrentHashMap<UUID, Values> pending;
private SpoutOutputCollector collector;
private String[] sentences = {
- "my dog has fleas",
- "i like cold beverages",
- "the dog ate my homework",
- "don't have a cow man",
- "i don't think i like fleas"
+ "my dog has fleas",
+ "i like cold beverages",
+ "the dog ate my homework",
+ "don't have a cow man",
+ "i don't think i like fleas"
};
private int index = 0;
private int count = 0;
@@ -151,7 +145,7 @@ public class SequenceFileTopology {
}
count++;
total++;
- if(count > 20000){
+ if (count > 20000) {
count = 0;
System.out.println("Pending count: " + this.pending.size() + ", total: " + this.total);
}
@@ -159,7 +153,7 @@ public class SequenceFileTopology {
}
public void ack(Object msgId) {
-// System.out.println("ACK");
+ // System.out.println("ACK");
this.pending.remove(msgId);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8ffac45/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java
index 943ca60..e3282d8 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/spout/HdfsSpoutTopology.java
@@ -1,192 +1,187 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
+import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
+import org.apache.storm.generated.ClusterSummary;
+import org.apache.storm.generated.ExecutorSummary;
import org.apache.storm.generated.KillOptions;
import org.apache.storm.generated.Nimbus;
+import org.apache.storm.generated.SpoutStats;
import org.apache.storm.generated.TopologyInfo;
import org.apache.storm.generated.TopologySummary;
-import org.apache.storm.generated.ExecutorSummary;
-import org.apache.storm.generated.SpoutStats;
-import org.apache.storm.generated.ClusterSummary;
import org.apache.storm.metric.LoggingMetricsConsumer;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.TopologyBuilder;
-import org.apache.storm.utils.Utils;
-import org.apache.storm.utils.NimbusClient;
import org.apache.storm.topology.base.BaseRichBolt;
-import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
+import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
-
public class HdfsSpoutTopology {
- public static final String SPOUT_ID = "hdfsspout";
- public static final String BOLT_ID = "constbolt";
-
+ public static final String SPOUT_ID = "hdfsspout";
+ public static final String BOLT_ID = "constbolt";
+
+ /**
+ * Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir after its done consuming
+ */
+ public static void main(String[] args) throws Exception {
+ // 0 - validate args
+ if (args.length < 7) {
+ System.err.println("Please check command line arguments.");
+ System.err.println("Usage :");
+ System.err.println(
+ HdfsSpoutTopology.class.toString() + " topologyName hdfsUri fileFormat sourceDir sourceArchiveDir badDir destinationDir.");
+ System.err.println(" topologyName - topology name.");
+ System.err.println(" hdfsUri - hdfs name node URI");
+ System.err.println(" fileFormat - Set to 'TEXT' for reading text files or 'SEQ' for sequence files.");
+ System.err.println(" sourceDir - read files from this HDFS dir using HdfsSpout.");
+ System.err.println(" archiveDir - after a file in sourceDir is read completely, it is moved to this HDFS location.");
+ System.err.println(" badDir - files that cannot be read properly will be moved to this HDFS location.");
+ System.err.println(" spoutCount - Num of spout instances.");
+ System.err.println();
+ System.exit(-1);
+ }
- public static class ConstBolt extends BaseRichBolt {
- private static final long serialVersionUID = -5313598399155365865L;
- public static final String FIELDS = "message";
- private OutputCollector collector;
- private static final Logger log = LoggerFactory.getLogger(ConstBolt.class);
- int count =0;
+ // 1 - parse cmd line args
+ String topologyName = args[0];
+ String hdfsUri = args[1];
+ String fileFormat = args[2];
+ String sourceDir = args[3];
+ String archiveDir = args[4];
+ String badDir = args[5];
+ int spoutNum = Integer.parseInt(args[6]);
+
+ // 2 - create and configure spout and bolt
+ ConstBolt bolt = new ConstBolt();
+
+ HdfsSpout spout = new HdfsSpout().withOutputFields(TextFileReader.defaultFields)
+ .setReaderType(fileFormat)
+ .setHdfsUri(hdfsUri)
+ .setSourceDir(sourceDir)
+ .setArchiveDir(archiveDir)
+ .setBadFilesDir(badDir);
+
+ // 3 - Create and configure topology
+ Config conf = new Config();
+ conf.setNumWorkers(1);
+ conf.setNumAckers(1);
+ conf.setMaxTaskParallelism(1);
+ conf.setDebug(true);
+ conf.registerMetricsConsumer(LoggingMetricsConsumer.class);
+
+ TopologyBuilder builder = new TopologyBuilder();
+ builder.setSpout(SPOUT_ID, spout, spoutNum);
+ builder.setBolt(BOLT_ID, bolt, 1).shuffleGrouping(SPOUT_ID);
+
+ // 4 - submit topology, wait for a few min and terminate it
+ Map<String, Object> clusterConf = Utils.readStormConfig();
+ StormSubmitter.submitTopologyWithProgressBar(topologyName, conf, builder.createTopology());
+ Nimbus.Iface client = NimbusClient.getConfiguredClient(clusterConf).getClient();
+
+ // 5 - Print metrics every 30 sec, kill topology after 20 min
+ for (int i = 0; i < 40; i++) {
+ Thread.sleep(30 * 1000);
+ printMetrics(client, topologyName);
+ }
+ kill(client, topologyName);
+ } // main
- public ConstBolt() {
+ private static void kill(Nimbus.Iface client, String topologyName) throws Exception {
+ KillOptions opts = new KillOptions();
+ opts.set_wait_secs(0);
+ client.killTopologyWithOpts(topologyName, opts);
}
- @Override
- public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
- this.collector = collector;
+ static void printMetrics(Nimbus.Iface client, String name) throws Exception {
+ ClusterSummary summary = client.getClusterInfo();
+ String id = null;
+ for (TopologySummary ts : summary.get_topologies()) {
+ if (name.equals(ts.get_name())) {
+ id = ts.get_id();
+ }
+ }
+ if (id == null) {
+ throw new Exception("Could not find a topology named " + name);
+ }
+ TopologyInfo info = client.getTopologyInfo(id);
+ int uptime = info.get_uptime_secs();
+ long acked = 0;
+ long failed = 0;
+ double weightedAvgTotal = 0.0;
+ for (ExecutorSummary exec : info.get_executors()) {
+ if ("spout".equals(exec.get_component_id())) {
+ SpoutStats stats = exec.get_stats().get_specific().get_spout();
+ Map<String, Long> failedMap = stats.get_failed().get(":all-time");
+ Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
+ Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
+ for (String key : ackedMap.keySet()) {
+ if (failedMap != null) {
+ Long tmp = failedMap.get(key);
+ if (tmp != null) {
+ failed += tmp;
+ }
+ }
+ long ackVal = ackedMap.get(key);
+ double latVal = avgLatMap.get(key) * ackVal;
+ acked += ackVal;
+ weightedAvgTotal += latVal;
+ }
+ }
+ }
+ double avgLatency = weightedAvgTotal / acked;
+ System.out.println("uptime: " + uptime + " acked: " + acked + " avgLatency: " + avgLatency + " acked/sec: " +
+ (((double) acked) / uptime + " failed: " + failed));
}
- @Override
- public void execute(Tuple tuple) {
- log.info("Received tuple : {}", tuple.getValue(0));
- count++;
- if(count==3) {
- collector.fail(tuple);
- }
- else {
- collector.ack(tuple);
- }
- }
+ public static class ConstBolt extends BaseRichBolt {
+ public static final String FIELDS = "message";
+ private static final long serialVersionUID = -5313598399155365865L;
+ private static final Logger log = LoggerFactory.getLogger(ConstBolt.class);
+ int count = 0;
+ private OutputCollector collector;
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields(FIELDS));
- }
- } // class
-
- /** Copies text file content from sourceDir to destinationDir. Moves source files into sourceDir after its done consuming */
- public static void main(String[] args) throws Exception {
- // 0 - validate args
- if (args.length < 7) {
- System.err.println("Please check command line arguments.");
- System.err.println("Usage :");
- System.err.println(HdfsSpoutTopology.class.toString() + " topologyName hdfsUri fileFormat sourceDir sourceArchiveDir badDir destinationDir.");
- System.err.println(" topologyName - topology name.");
- System.err.println(" hdfsUri - hdfs name node URI");
- System.err.println(" fileFormat - Set to 'TEXT' for reading text files or 'SEQ' for sequence files.");
- System.err.println(" sourceDir - read files from this HDFS dir using HdfsSpout.");
- System.err.println(" archiveDir - after a file in sourceDir is read completely, it is moved to this HDFS location.");
- System.err.println(" badDir - files that cannot be read properly will be moved to this HDFS location.");
- System.err.println(" spoutCount - Num of spout instances.");
- System.err.println();
- System.exit(-1);
- }
+ public ConstBolt() {
+ }
- // 1 - parse cmd line args
- String topologyName = args[0];
- String hdfsUri = args[1];
- String fileFormat = args[2];
- String sourceDir = args[3];
- String archiveDir = args[4];
- String badDir = args[5];
- int spoutNum = Integer.parseInt(args[6]);
-
- // 2 - create and configure spout and bolt
- ConstBolt bolt = new ConstBolt();
-
- HdfsSpout spout = new HdfsSpout().withOutputFields(TextFileReader.defaultFields)
- .setReaderType(fileFormat)
- .setHdfsUri(hdfsUri)
- .setSourceDir(sourceDir)
- .setArchiveDir(archiveDir)
- .setBadFilesDir(badDir);
-
- // 3 - Create and configure topology
- Config conf = new Config();
- conf.setNumWorkers(1);
- conf.setNumAckers(1);
- conf.setMaxTaskParallelism(1);
- conf.setDebug(true);
- conf.registerMetricsConsumer(LoggingMetricsConsumer.class);
-
- TopologyBuilder builder = new TopologyBuilder();
- builder.setSpout(SPOUT_ID, spout, spoutNum);
- builder.setBolt(BOLT_ID, bolt, 1).shuffleGrouping(SPOUT_ID);
-
- // 4 - submit topology, wait for a few min and terminate it
- Map<String, Object> clusterConf = Utils.readStormConfig();
- StormSubmitter.submitTopologyWithProgressBar(topologyName, conf, builder.createTopology());
- Nimbus.Iface client = NimbusClient.getConfiguredClient(clusterConf).getClient();
-
- // 5 - Print metrics every 30 sec, kill topology after 20 min
- for (int i = 0; i < 40; i++) {
- Thread.sleep(30 * 1000);
- printMetrics(client, topologyName);
- }
- kill(client, topologyName);
- } // main
-
- private static void kill(Nimbus.Iface client, String topologyName) throws Exception {
- KillOptions opts = new KillOptions();
- opts.set_wait_secs(0);
- client.killTopologyWithOpts(topologyName, opts);
- }
-
- static void printMetrics(Nimbus.Iface client, String name) throws Exception {
- ClusterSummary summary = client.getClusterInfo();
- String id = null;
- for (TopologySummary ts: summary.get_topologies()) {
- if (name.equals(ts.get_name())) {
- id = ts.get_id();
- }
- }
- if (id == null) {
- throw new Exception("Could not find a topology named "+name);
- }
- TopologyInfo info = client.getTopologyInfo(id);
- int uptime = info.get_uptime_secs();
- long acked = 0;
- long failed = 0;
- double weightedAvgTotal = 0.0;
- for (ExecutorSummary exec: info.get_executors()) {
- if ("spout".equals(exec.get_component_id())) {
- SpoutStats stats = exec.get_stats().get_specific().get_spout();
- Map<String, Long> failedMap = stats.get_failed().get(":all-time");
- Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
- Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
- for (String key: ackedMap.keySet()) {
- if (failedMap != null) {
- Long tmp = failedMap.get(key);
- if (tmp != null) {
- failed += tmp;
+ @Override
+ public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
+ this.collector = collector;
+ }
+
+ @Override
+ public void execute(Tuple tuple) {
+ log.info("Received tuple : {}", tuple.getValue(0));
+ count++;
+ if (count == 3) {
+ collector.fail(tuple);
+ } else {
+ collector.ack(tuple);
}
- }
- long ackVal = ackedMap.get(key);
- double latVal = avgLatMap.get(key) * ackVal;
- acked += ackVal;
- weightedAvgTotal += latVal;
}
- }
- }
- double avgLatency = weightedAvgTotal/acked;
- System.out.println("uptime: "+uptime+" acked: "+acked+" avgLatency: "+avgLatency+" acked/sec: "+(((double)acked)/uptime+" failed: "+failed));
- }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields(FIELDS));
+ }
+ } // class
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8ffac45/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/FixedBatchSpout.java
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/FixedBatchSpout.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/FixedBatchSpout.java
index 1553971..29793f1 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/FixedBatchSpout.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/FixedBatchSpout.java
@@ -15,18 +15,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.hdfs.trident;
-import org.apache.storm.Config;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.trident.operation.TridentCollector;
-import org.apache.storm.trident.spout.IBatchSpout;
+package org.apache.storm.hdfs.trident;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.storm.Config;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.spout.IBatchSpout;
+import org.apache.storm.tuple.Fields;
public class FixedBatchSpout implements IBatchSpout {
@@ -34,6 +34,8 @@ public class FixedBatchSpout implements IBatchSpout {
List<Object>[] outputs;
int maxBatchSize;
HashMap<Long, List<List<Object>>> batches = new HashMap<Long, List<List<Object>>>();
+ int index = 0;
+ boolean cycle = false;
public FixedBatchSpout(Fields fields, int maxBatchSize, List<Object>... outputs) {
this.fields = fields;
@@ -41,9 +43,6 @@ public class FixedBatchSpout implements IBatchSpout {
this.maxBatchSize = maxBatchSize;
}
- int index = 0;
- boolean cycle = false;
-
public void setCycle(boolean cycle) {
this.cycle = cycle;
}
@@ -56,20 +55,20 @@ public class FixedBatchSpout implements IBatchSpout {
@Override
public void emitBatch(long batchId, TridentCollector collector) {
List<List<Object>> batch = this.batches.get(batchId);
- if(batch == null){
+ if (batch == null) {
batch = new ArrayList<List<Object>>();
- if(index>=outputs.length && cycle) {
+ if (index >= outputs.length && cycle) {
index = 0;
}
- for(int i=0; i < maxBatchSize; index++, i++) {
- if(index == outputs.length){
- index=0;
+ for (int i = 0; i < maxBatchSize; index++, i++) {
+ if (index == outputs.length) {
+ index = 0;
}
batch.add(outputs[index]);
}
this.batches.put(batchId, batch);
}
- for(List<Object> list : batch){
+ for (List<Object> list : batch) {
collector.emit(list);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8ffac45/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java
index af76c00..e8bf490 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentFileTopology.java
@@ -15,12 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.trident;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -40,10 +40,12 @@ import org.yaml.snakeyaml.Yaml;
public class TridentFileTopology {
- public static StormTopology buildTopology(String hdfsUrl){
+ public static StormTopology buildTopology(String hdfsUrl) {
FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
- new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
- new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
+ new Values("the man went to the store and bought some candy", 2l),
+ new Values("four score and seven years ago", 3l),
+ new Values("how many apples can you eat", 4l),
+ new Values("to be or not to be the person", 5l));
spout.setCycle(true);
TridentTopology topology = new TridentTopology();
@@ -52,26 +54,26 @@ public class TridentFileTopology {
Fields hdfsFields = new Fields("sentence", "key");
FileNameFormat fileNameFormat = new DefaultFileNameFormat()
- .withPath("/tmp/trident")
- .withPrefix("trident")
- .withExtension(".txt");
+ .withPath("/tmp/trident")
+ .withPrefix("trident")
+ .withExtension(".txt");
RecordFormat recordFormat = new DelimitedRecordFormat()
- .withFields(hdfsFields);
+ .withFields(hdfsFields);
FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
HdfsState.Options options = new HdfsState.HdfsFileOptions()
- .withFileNameFormat(fileNameFormat)
- .withRecordFormat(recordFormat)
- .withRotationPolicy(rotationPolicy)
- .withFsUrl(hdfsUrl)
- .withConfigKey("hdfs.config");
+ .withFileNameFormat(fileNameFormat)
+ .withRecordFormat(recordFormat)
+ .withRotationPolicy(rotationPolicy)
+ .withFsUrl(hdfsUrl)
+ .withConfigKey("hdfs.config");
StateFactory factory = new HdfsStateFactory().withOptions(options);
TridentState state = stream
- .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());
+ .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());
return topology.build();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8ffac45/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java
index 525770b..15612bd 100644
--- a/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java
+++ b/examples/storm-hdfs-examples/src/main/java/org/apache/storm/hdfs/trident/TridentSequenceTopology.java
@@ -15,12 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.trident;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -40,10 +40,12 @@ import org.yaml.snakeyaml.Yaml;
public class TridentSequenceTopology {
- public static StormTopology buildTopology(String hdfsUrl){
+ public static StormTopology buildTopology(String hdfsUrl) {
FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l),
- new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l),
- new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
+ new Values("the man went to the store and bought some candy", 2l),
+ new Values("four score and seven years ago", 3l),
+ new Values("how many apples can you eat", 4l),
+ new Values("to be or not to be the person", 5l));
spout.setCycle(true);
TridentTopology topology = new TridentTopology();
@@ -52,23 +54,23 @@ public class TridentSequenceTopology {
Fields hdfsFields = new Fields("sentence", "key");
FileNameFormat fileNameFormat = new DefaultFileNameFormat()
- .withPath("/tmp/trident")
- .withPrefix("trident")
- .withExtension(".seq");
+ .withPath("/tmp/trident")
+ .withPrefix("trident")
+ .withExtension(".seq");
FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
HdfsState.Options seqOpts = new HdfsState.SequenceFileOptions()
- .withFileNameFormat(fileNameFormat)
- .withSequenceFormat(new DefaultSequenceFormat("key", "sentence"))
- .withRotationPolicy(rotationPolicy)
- .withFsUrl(hdfsUrl)
- .withConfigKey("hdfs.config")
- .addRotationAction(new MoveFileAction().toDestination("/tmp/dest2/"));
+ .withFileNameFormat(fileNameFormat)
+ .withSequenceFormat(new DefaultSequenceFormat("key", "sentence"))
+ .withRotationPolicy(rotationPolicy)
+ .withFsUrl(hdfsUrl)
+ .withConfigKey("hdfs.config")
+ .addRotationAction(new MoveFileAction().toDestination("/tmp/dest2/"));
StateFactory factory = new HdfsStateFactory().withOptions(seqOpts);
TridentState state = stream
- .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());
+ .partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());
return topology.build();
}
[03/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java
index b70e844..f20daf5 100644
--- a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java
+++ b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/BasicContainerTest.java
@@ -1,24 +1,16 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.daemon.supervisor;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+package org.apache.storm.daemon.supervisor;
import java.io.File;
import java.io.IOException;
@@ -28,7 +20,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
import org.apache.storm.container.ResourceIsolationInterface;
@@ -37,84 +28,63 @@ import org.apache.storm.generated.LocalAssignment;
import org.apache.storm.generated.ProfileAction;
import org.apache.storm.generated.ProfileRequest;
import org.apache.storm.generated.StormTopology;
+import org.apache.storm.utils.LocalState;
import org.apache.storm.utils.SimpleVersion;
import org.apache.storm.utils.Utils;
-import org.apache.storm.utils.LocalState;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
public class BasicContainerTest {
- public static class CommandRun {
- final List<String> cmd;
- final Map<String, String> env;
- final File pwd;
-
- public CommandRun(List<String> cmd, Map<String, String> env, File pwd) {
- this.cmd = cmd;
- this.env = env;
- this.pwd = pwd;
+ private static void setSystemProp(String key, String value) {
+ if (value == null) {
+ System.clearProperty(key);
+ } else {
+ System.setProperty(key, value);
}
}
-
- public static class MockBasicContainer extends BasicContainer {
- public MockBasicContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int supervisorPort,
- int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager,
- LocalState localState, String workerId, Map<String, Object> topoConf, AdvancedFSOps ops,
- String profileCmd) throws IOException {
- super(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, localState,
- workerId, topoConf, ops, profileCmd);
- }
-
- public final List<CommandRun> profileCmds = new ArrayList<>();
- public final List<CommandRun> workerCmds = new ArrayList<>();
-
- @Override
- protected Map<String, Object> readTopoConf() throws IOException {
- return new HashMap<>();
- }
-
- @Override
- public void createNewWorkerId() {
- super.createNewWorkerId();
- }
-
- @Override
- public List<String> substituteChildopts(Object value, int memOnheap) {
- return super.substituteChildopts(value, memOnheap);
- }
-
- @Override
- protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix,
- File targetDir) throws IOException, InterruptedException {
- profileCmds.add(new CommandRun(command, env, targetDir));
- return true;
+ private static void checkpoint(Run r, String... newValues) throws Exception {
+ if (newValues.length % 2 != 0) {
+ throw new IllegalArgumentException("Parameters are of the form system property name, new value");
}
-
- @Override
- protected void launchWorkerProcess(List<String> command, Map<String, String> env, String logPrefix,
- ExitCodeCallback processExitCallback, File targetDir) throws IOException {
- workerCmds.add(new CommandRun(command, env, targetDir));
+ Map<String, String> orig = new HashMap<>();
+ try {
+ for (int index = 0; index < newValues.length; index += 2) {
+ String key = newValues[index];
+ String value = newValues[index + 1];
+ orig.put(key, System.getProperty(key));
+ setSystemProp(key, value);
+ }
+ r.run();
+ } finally {
+ for (Map.Entry<String, String> entry : orig.entrySet()) {
+ setSystemProp(entry.getKey(), entry.getValue());
+ }
}
-
- @Override
- protected String javaCmd(String cmd) {
- //avoid system dependent things
- return cmd;
+ }
+
+ private static <T> void assertListEquals(List<T> a, List<T> b) {
+ if (a == null) {
+ assertNull(b);
}
-
- @Override
- protected List<String> frameworkClasspath(SimpleVersion version) {
- //We are not really running anything so make this
- // simple to check for
- return Arrays.asList("FRAMEWORK_CP");
+ if (b == null) {
+ assertNull(a);
}
-
- @Override
- protected String javaLibraryPath(String stormRoot, Map<String, Object> conf) {
- return "JLP";
+ int commonLen = Math.min(a.size(), b.size());
+ for (int i = 0; i < commonLen; i++) {
+ assertEquals("at index " + i + "\n" + a + " !=\n" + b + "\n", a.get(i), b.get(i));
}
+
+ assertEquals("size of lists don't match \n" + a + " !=\n" + b, a.size(), b.size());
}
-
+
@Test
public void testCreateNewWorkerId() throws Exception {
final String topoId = "test_topology";
@@ -122,24 +92,25 @@ public class BasicContainerTest {
final int port = 8080;
LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
-
+
Map<String, Object> superConf = new HashMap<>();
AdvancedFSOps ops = mock(AdvancedFSOps.class);
when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
-
+
LocalState ls = mock(LocalState.class);
-
- MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", supervisorPort, port, la, null, ls, null, new HashMap<>(), ops, "profile");
+
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", supervisorPort, port, la, null, ls, null, new HashMap<>(), ops,
+ "profile");
//null worker id means generate one...
-
+
assertNotNull(mc._workerId);
verify(ls).getApprovedWorkers();
Map<String, Integer> expectedNewState = new HashMap<String, Integer>();
expectedNewState.put(mc._workerId, port);
verify(ls).setApprovedWorkers(expectedNewState);
}
-
+
@Test
public void testRecovery() throws Exception {
final String topoId = "test_topology";
@@ -148,23 +119,24 @@ public class BasicContainerTest {
final int port = 8080;
LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
-
+
Map<String, Integer> workerState = new HashMap<String, Integer>();
workerState.put(workerId, port);
-
+
LocalState ls = mock(LocalState.class);
when(ls.getApprovedWorkers()).thenReturn(workerState);
-
+
Map<String, Object> superConf = new HashMap<>();
AdvancedFSOps ops = mock(AdvancedFSOps.class);
when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
-
- MockBasicContainer mc = new MockBasicContainer(ContainerType.RECOVER_FULL, superConf,
- "SUPERVISOR", supervisorPort, port, la, null, ls, null, new HashMap<>(), ops, "profile");
-
+
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.RECOVER_FULL, superConf,
+ "SUPERVISOR", supervisorPort, port, la, null, ls, null, new HashMap<>(), ops,
+ "profile");
+
assertEquals(workerId, mc._workerId);
}
-
+
@Test
public void testRecoveryMiss() throws Exception {
final String topoId = "test_topology";
@@ -172,22 +144,22 @@ public class BasicContainerTest {
final int port = 8080;
LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
-
+
Map<String, Integer> workerState = new HashMap<String, Integer>();
- workerState.put("somethingelse", port+1);
-
+ workerState.put("somethingelse", port + 1);
+
LocalState ls = mock(LocalState.class);
when(ls.getApprovedWorkers()).thenReturn(workerState);
-
+
try {
- new MockBasicContainer(ContainerType.RECOVER_FULL, new HashMap<String, Object>(),
- "SUPERVISOR", supervisorPort, port, la, null, ls, null, new HashMap<>(), null, "profile");
+ new MockBasicContainer(ContainerType.RECOVER_FULL, new HashMap<String, Object>(),
+ "SUPERVISOR", supervisorPort, port, la, null, ls, null, new HashMap<>(), null, "profile");
fail("Container recovered worker incorrectly");
} catch (ContainerRecoveryException e) {
//Expected
}
}
-
+
@Test
public void testCleanUp() throws Exception {
final String topoId = "test_topology";
@@ -196,28 +168,29 @@ public class BasicContainerTest {
final String workerId = "worker-id";
LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
-
+
Map<String, Object> superConf = new HashMap<>();
AdvancedFSOps ops = mock(AdvancedFSOps.class);
when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
-
+
Map<String, Integer> workerState = new HashMap<String, Integer>();
workerState.put(workerId, port);
-
+
LocalState ls = mock(LocalState.class);
when(ls.getApprovedWorkers()).thenReturn(new HashMap<>(workerState));
-
- MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new HashMap<>(), ops, "profile");
-
+
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new HashMap<>(), ops,
+ "profile");
+
mc.cleanUp();
-
+
assertNull(mc._workerId);
verify(ls).getApprovedWorkers();
Map<String, Integer> expectedNewState = new HashMap<String, Integer>();
verify(ls).setApprovedWorkers(expectedNewState);
}
-
+
@Test
public void testRunProfiling() throws Exception {
final long pid = 100;
@@ -228,138 +201,92 @@ public class BasicContainerTest {
final String stormLocal = ContainerTest.asAbsPath("tmp", "testing");
final String topoRoot = ContainerTest.asAbsPath(stormLocal, topoId, String.valueOf(port));
final File workerArtifactsPid = ContainerTest.asAbsFile(topoRoot, "worker.pid");
-
+
final Map<String, Object> superConf = new HashMap<>();
superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
-
+
LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
-
+
AdvancedFSOps ops = mock(AdvancedFSOps.class);
when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
when(ops.slurpString(workerArtifactsPid)).thenReturn(String.valueOf(pid));
-
+
LocalState ls = mock(LocalState.class);
-
- MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new HashMap<>(), ops, "profile");
-
+
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new HashMap<>(), ops,
+ "profile");
+
//HEAP DUMP
ProfileRequest req = new ProfileRequest();
req.set_action(ProfileAction.JMAP_DUMP);
-
+
mc.runProfiling(req, false);
-
+
assertEquals(1, mc.profileCmds.size());
CommandRun cmd = mc.profileCmds.get(0);
mc.profileCmds.clear();
assertEquals(Arrays.asList("profile", String.valueOf(pid), "jmap", topoRoot), cmd.cmd);
assertEquals(new File(topoRoot), cmd.pwd);
-
+
//JSTACK DUMP
req.set_action(ProfileAction.JSTACK_DUMP);
-
+
mc.runProfiling(req, false);
-
+
assertEquals(1, mc.profileCmds.size());
cmd = mc.profileCmds.get(0);
mc.profileCmds.clear();
assertEquals(Arrays.asList("profile", String.valueOf(pid), "jstack", topoRoot), cmd.cmd);
assertEquals(new File(topoRoot), cmd.pwd);
-
+
//RESTART
req.set_action(ProfileAction.JVM_RESTART);
-
+
mc.runProfiling(req, false);
-
+
assertEquals(1, mc.profileCmds.size());
cmd = mc.profileCmds.get(0);
mc.profileCmds.clear();
assertEquals(Arrays.asList("profile", String.valueOf(pid), "kill"), cmd.cmd);
assertEquals(new File(topoRoot), cmd.pwd);
-
+
//JPROFILE DUMP
req.set_action(ProfileAction.JPROFILE_DUMP);
-
+
mc.runProfiling(req, false);
-
+
assertEquals(1, mc.profileCmds.size());
cmd = mc.profileCmds.get(0);
mc.profileCmds.clear();
assertEquals(Arrays.asList("profile", String.valueOf(pid), "dump", topoRoot), cmd.cmd);
assertEquals(new File(topoRoot), cmd.pwd);
-
+
//JPROFILE START
req.set_action(ProfileAction.JPROFILE_STOP);
-
+
mc.runProfiling(req, false);
-
+
assertEquals(1, mc.profileCmds.size());
cmd = mc.profileCmds.get(0);
mc.profileCmds.clear();
assertEquals(Arrays.asList("profile", String.valueOf(pid), "start"), cmd.cmd);
assertEquals(new File(topoRoot), cmd.pwd);
-
+
//JPROFILE STOP
req.set_action(ProfileAction.JPROFILE_STOP);
-
+
mc.runProfiling(req, true);
-
+
assertEquals(1, mc.profileCmds.size());
cmd = mc.profileCmds.get(0);
mc.profileCmds.clear();
assertEquals(Arrays.asList("profile", String.valueOf(pid), "stop", topoRoot), cmd.cmd);
assertEquals(new File(topoRoot), cmd.pwd);
}
-
- private static void setSystemProp(String key, String value) {
- if (value == null) {
- System.clearProperty(key);
- } else {
- System.setProperty(key, value);
- }
- }
-
- private static interface Run {
- public void run() throws Exception;
- }
-
- private static void checkpoint(Run r, String ... newValues) throws Exception {
- if (newValues.length % 2 != 0) {
- throw new IllegalArgumentException("Parameters are of the form system property name, new value");
- }
- Map<String, String> orig = new HashMap<>();
- try {
- for (int index = 0; index < newValues.length; index += 2) {
- String key = newValues[index];
- String value = newValues[index + 1];
- orig.put(key, System.getProperty(key));
- setSystemProp(key, value);
- }
- r.run();
- } finally {
- for (Map.Entry<String, String> entry: orig.entrySet()) {
- setSystemProp(entry.getKey(), entry.getValue());
- }
- }
- }
-
- private static <T> void assertListEquals(List<T> a, List<T> b) {
- if (a == null) {
- assertNull(b);
- }
- if (b == null) {
- assertNull(a);
- }
- int commonLen = Math.min(a.size(), b.size());
- for (int i = 0; i < commonLen; i++) {
- assertEquals("at index "+i+"\n"+a+" !=\n"+b+"\n", a.get(i), b.get(i));
- }
-
- assertEquals("size of lists don't match \n"+a+" !=\n"+b, a.size(), b.size());
- }
-
+
@Test
public void testLaunch() throws Exception {
final String topoId = "test_topology_current";
@@ -376,86 +303,88 @@ public class BasicContainerTest {
final String workerConf = ContainerTest.asAbsPath(log4jdir, "worker.xml");
final String workerRoot = ContainerTest.asAbsPath(stormLocal, "workers", workerId);
final String workerTmpDir = ContainerTest.asAbsPath(workerRoot, "tmp");
-
+
final StormTopology st = new StormTopology();
st.set_spouts(new HashMap<>());
st.set_bolts(new HashMap<>());
st.set_state_spouts(new HashMap<>());
- byte [] serializedState = Utils.gzip(Utils.thriftSerialize(st));
-
+ byte[] serializedState = Utils.gzip(Utils.thriftSerialize(st));
+
final Map<String, Object> superConf = new HashMap<>();
superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
superConf.put(DaemonConfig.STORM_LOG4J2_CONF_DIR, log4jdir);
superConf.put(Config.WORKER_CHILDOPTS, " -Dtesting=true");
-
+
LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
-
+
AdvancedFSOps ops = mock(AdvancedFSOps.class);
when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
when(ops.slurp(stormcode)).thenReturn(serializedState);
-
+
LocalState ls = mock(LocalState.class);
-
+
checkpoint(() -> {
- MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new HashMap<>(), ops, "profile");
-
- mc.launch();
-
- assertEquals(1, mc.workerCmds.size());
- CommandRun cmd = mc.workerCmds.get(0);
- mc.workerCmds.clear();
- assertListEquals(Arrays.asList(
- "java",
- "-cp",
- "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
- "-Dlogging.sensitivity=S3",
- "-Dlogfile.name=worker.log",
- "-Dstorm.home=" + stormHome,
- "-Dworkers.artifacts=" + stormLocal,
- "-Dstorm.id=" + topoId,
- "-Dworker.id=" + workerId,
- "-Dworker.port=" + port,
- "-Dstorm.log.dir=" + stormLogDir,
- "-Dlog4j.configurationFile=" + workerConf,
- "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
- "-Dstorm.local.dir=" + stormLocal,
- "-Dworker.memory_limit_mb=768",
- "org.apache.storm.LogWriter",
- "java",
- "-server",
- "-Dlogging.sensitivity=S3",
- "-Dlogfile.name=worker.log",
- "-Dstorm.home=" + stormHome,
- "-Dworkers.artifacts=" + stormLocal,
- "-Dstorm.id=" + topoId,
- "-Dworker.id=" + workerId,
- "-Dworker.port=" + port,
- "-Dstorm.log.dir=" + stormLogDir,
- "-Dlog4j.configurationFile=" + workerConf,
- "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
- "-Dstorm.local.dir=" + stormLocal,
- "-Dworker.memory_limit_mb=768",
- "-Dtesting=true",
- "-Djava.library.path=JLP",
- "-Dstorm.conf.file=",
- "-Dstorm.options=",
- "-Djava.io.tmpdir="+workerTmpDir,
- "-cp",
- "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
- "org.apache.storm.daemon.worker.Worker",
- topoId,
- "SUPERVISOR",
- String.valueOf(supervisorPort),
- String.valueOf(port),
- workerId
- ), cmd.cmd);
- assertEquals(new File(workerRoot), cmd.pwd);
- },
- "storm.home", stormHome,
- "storm.log.dir", stormLogDir);
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new
+ HashMap<>(), ops,
+ "profile");
+
+ mc.launch();
+
+ assertEquals(1, mc.workerCmds.size());
+ CommandRun cmd = mc.workerCmds.get(0);
+ mc.workerCmds.clear();
+ assertListEquals(Arrays.asList(
+ "java",
+ "-cp",
+ "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
+ "-Dlogging.sensitivity=S3",
+ "-Dlogfile.name=worker.log",
+ "-Dstorm.home=" + stormHome,
+ "-Dworkers.artifacts=" + stormLocal,
+ "-Dstorm.id=" + topoId,
+ "-Dworker.id=" + workerId,
+ "-Dworker.port=" + port,
+ "-Dstorm.log.dir=" + stormLogDir,
+ "-Dlog4j.configurationFile=" + workerConf,
+ "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
+ "-Dstorm.local.dir=" + stormLocal,
+ "-Dworker.memory_limit_mb=768",
+ "org.apache.storm.LogWriter",
+ "java",
+ "-server",
+ "-Dlogging.sensitivity=S3",
+ "-Dlogfile.name=worker.log",
+ "-Dstorm.home=" + stormHome,
+ "-Dworkers.artifacts=" + stormLocal,
+ "-Dstorm.id=" + topoId,
+ "-Dworker.id=" + workerId,
+ "-Dworker.port=" + port,
+ "-Dstorm.log.dir=" + stormLogDir,
+ "-Dlog4j.configurationFile=" + workerConf,
+ "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
+ "-Dstorm.local.dir=" + stormLocal,
+ "-Dworker.memory_limit_mb=768",
+ "-Dtesting=true",
+ "-Djava.library.path=JLP",
+ "-Dstorm.conf.file=",
+ "-Dstorm.options=",
+ "-Djava.io.tmpdir=" + workerTmpDir,
+ "-cp",
+ "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
+ "org.apache.storm.daemon.worker.Worker",
+ topoId,
+ "SUPERVISOR",
+ String.valueOf(supervisorPort),
+ String.valueOf(port),
+ workerId
+ ), cmd.cmd);
+ assertEquals(new File(workerRoot), cmd.pwd);
+ },
+ "storm.home", stormHome,
+ "storm.log.dir", stormLogDir);
}
@Test
@@ -482,7 +411,7 @@ public class BasicContainerTest {
// minimum 1.x version of supporting STORM-2448 would be 1.0.4
st.set_storm_version("1.0.4");
- byte [] serializedState = Utils.gzip(Utils.thriftSerialize(st));
+ byte[] serializedState = Utils.gzip(Utils.thriftSerialize(st));
final Map<String, Object> superConf = new HashMap<>();
superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
@@ -500,62 +429,64 @@ public class BasicContainerTest {
LocalState ls = mock(LocalState.class);
checkpoint(() -> {
- MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new HashMap<>(), ops, "profile");
-
- mc.launch();
-
- assertEquals(1, mc.workerCmds.size());
- CommandRun cmd = mc.workerCmds.get(0);
- mc.workerCmds.clear();
- assertListEquals(Arrays.asList(
- "java",
- "-cp",
- "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
- "-Dlogging.sensitivity=S3",
- "-Dlogfile.name=worker.log",
- "-Dstorm.home=" + stormHome,
- "-Dworkers.artifacts=" + stormLocal,
- "-Dstorm.id=" + topoId,
- "-Dworker.id=" + workerId,
- "-Dworker.port=" + port,
- "-Dstorm.log.dir=" + stormLogDir,
- "-Dlog4j.configurationFile=" + workerConf,
- "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
- "-Dstorm.local.dir=" + stormLocal,
- "-Dworker.memory_limit_mb=768",
- "org.apache.storm.LogWriter",
- "java",
- "-server",
- "-Dlogging.sensitivity=S3",
- "-Dlogfile.name=worker.log",
- "-Dstorm.home=" + stormHome,
- "-Dworkers.artifacts=" + stormLocal,
- "-Dstorm.id=" + topoId,
- "-Dworker.id=" + workerId,
- "-Dworker.port=" + port,
- "-Dstorm.log.dir=" + stormLogDir,
- "-Dlog4j.configurationFile=" + workerConf,
- "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
- "-Dstorm.local.dir=" + stormLocal,
- "-Dworker.memory_limit_mb=768",
- "-Dtesting=true",
- "-Djava.library.path=JLP",
- "-Dstorm.conf.file=",
- "-Dstorm.options=",
- "-Djava.io.tmpdir="+workerTmpDir,
- "-cp",
- "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
- "org.apache.storm.daemon.worker",
- topoId,
- "SUPERVISOR",
- String.valueOf(port),
- workerId
- ), cmd.cmd);
- assertEquals(new File(workerRoot), cmd.pwd);
- },
- "storm.home", stormHome,
- "storm.log.dir", stormLogDir);
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new
+ HashMap<>(), ops,
+ "profile");
+
+ mc.launch();
+
+ assertEquals(1, mc.workerCmds.size());
+ CommandRun cmd = mc.workerCmds.get(0);
+ mc.workerCmds.clear();
+ assertListEquals(Arrays.asList(
+ "java",
+ "-cp",
+ "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
+ "-Dlogging.sensitivity=S3",
+ "-Dlogfile.name=worker.log",
+ "-Dstorm.home=" + stormHome,
+ "-Dworkers.artifacts=" + stormLocal,
+ "-Dstorm.id=" + topoId,
+ "-Dworker.id=" + workerId,
+ "-Dworker.port=" + port,
+ "-Dstorm.log.dir=" + stormLogDir,
+ "-Dlog4j.configurationFile=" + workerConf,
+ "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
+ "-Dstorm.local.dir=" + stormLocal,
+ "-Dworker.memory_limit_mb=768",
+ "org.apache.storm.LogWriter",
+ "java",
+ "-server",
+ "-Dlogging.sensitivity=S3",
+ "-Dlogfile.name=worker.log",
+ "-Dstorm.home=" + stormHome,
+ "-Dworkers.artifacts=" + stormLocal,
+ "-Dstorm.id=" + topoId,
+ "-Dworker.id=" + workerId,
+ "-Dworker.port=" + port,
+ "-Dstorm.log.dir=" + stormLogDir,
+ "-Dlog4j.configurationFile=" + workerConf,
+ "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
+ "-Dstorm.local.dir=" + stormLocal,
+ "-Dworker.memory_limit_mb=768",
+ "-Dtesting=true",
+ "-Djava.library.path=JLP",
+ "-Dstorm.conf.file=",
+ "-Dstorm.options=",
+ "-Djava.io.tmpdir=" + workerTmpDir,
+ "-cp",
+ "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
+ "org.apache.storm.daemon.worker",
+ topoId,
+ "SUPERVISOR",
+ String.valueOf(port),
+ workerId
+ ), cmd.cmd);
+ assertEquals(new File(workerRoot), cmd.pwd);
+ },
+ "storm.home", stormHome,
+ "storm.log.dir", stormLogDir);
}
@Test
@@ -582,7 +513,7 @@ public class BasicContainerTest {
// minimum 0.x version of supporting STORM-2448 would be 0.10.3
st.set_storm_version("0.10.3");
- byte [] serializedState = Utils.gzip(Utils.thriftSerialize(st));
+ byte[] serializedState = Utils.gzip(Utils.thriftSerialize(st));
final Map<String, Object> superConf = new HashMap<>();
superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
@@ -600,62 +531,64 @@ public class BasicContainerTest {
LocalState ls = mock(LocalState.class);
checkpoint(() -> {
- MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new HashMap<>(), ops, "profile");
-
- mc.launch();
-
- assertEquals(1, mc.workerCmds.size());
- CommandRun cmd = mc.workerCmds.get(0);
- mc.workerCmds.clear();
- assertListEquals(Arrays.asList(
- "java",
- "-cp",
- "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
- "-Dlogging.sensitivity=S3",
- "-Dlogfile.name=worker.log",
- "-Dstorm.home=" + stormHome,
- "-Dworkers.artifacts=" + stormLocal,
- "-Dstorm.id=" + topoId,
- "-Dworker.id=" + workerId,
- "-Dworker.port=" + port,
- "-Dstorm.log.dir=" + stormLogDir,
- "-Dlog4j.configurationFile=" + workerConf,
- "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
- "-Dstorm.local.dir=" + stormLocal,
- "-Dworker.memory_limit_mb=768",
- "backtype.storm.LogWriter",
- "java",
- "-server",
- "-Dlogging.sensitivity=S3",
- "-Dlogfile.name=worker.log",
- "-Dstorm.home=" + stormHome,
- "-Dworkers.artifacts=" + stormLocal,
- "-Dstorm.id=" + topoId,
- "-Dworker.id=" + workerId,
- "-Dworker.port=" + port,
- "-Dstorm.log.dir=" + stormLogDir,
- "-Dlog4j.configurationFile=" + workerConf,
- "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
- "-Dstorm.local.dir=" + stormLocal,
- "-Dworker.memory_limit_mb=768",
- "-Dtesting=true",
- "-Djava.library.path=JLP",
- "-Dstorm.conf.file=",
- "-Dstorm.options=",
- "-Djava.io.tmpdir="+workerTmpDir,
- "-cp",
- "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
- "backtype.storm.daemon.worker",
- topoId,
- "SUPERVISOR",
- String.valueOf(port),
- workerId
- ), cmd.cmd);
- assertEquals(new File(workerRoot), cmd.pwd);
- },
- "storm.home", stormHome,
- "storm.log.dir", stormLogDir);
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new
+ HashMap<>(), ops,
+ "profile");
+
+ mc.launch();
+
+ assertEquals(1, mc.workerCmds.size());
+ CommandRun cmd = mc.workerCmds.get(0);
+ mc.workerCmds.clear();
+ assertListEquals(Arrays.asList(
+ "java",
+ "-cp",
+ "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
+ "-Dlogging.sensitivity=S3",
+ "-Dlogfile.name=worker.log",
+ "-Dstorm.home=" + stormHome,
+ "-Dworkers.artifacts=" + stormLocal,
+ "-Dstorm.id=" + topoId,
+ "-Dworker.id=" + workerId,
+ "-Dworker.port=" + port,
+ "-Dstorm.log.dir=" + stormLogDir,
+ "-Dlog4j.configurationFile=" + workerConf,
+ "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
+ "-Dstorm.local.dir=" + stormLocal,
+ "-Dworker.memory_limit_mb=768",
+ "backtype.storm.LogWriter",
+ "java",
+ "-server",
+ "-Dlogging.sensitivity=S3",
+ "-Dlogfile.name=worker.log",
+ "-Dstorm.home=" + stormHome,
+ "-Dworkers.artifacts=" + stormLocal,
+ "-Dstorm.id=" + topoId,
+ "-Dworker.id=" + workerId,
+ "-Dworker.port=" + port,
+ "-Dstorm.log.dir=" + stormLogDir,
+ "-Dlog4j.configurationFile=" + workerConf,
+ "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
+ "-Dstorm.local.dir=" + stormLocal,
+ "-Dworker.memory_limit_mb=768",
+ "-Dtesting=true",
+ "-Djava.library.path=JLP",
+ "-Dstorm.conf.file=",
+ "-Dstorm.options=",
+ "-Djava.io.tmpdir=" + workerTmpDir,
+ "-cp",
+ "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
+ "backtype.storm.daemon.worker",
+ topoId,
+ "SUPERVISOR",
+ String.valueOf(port),
+ workerId
+ ), cmd.cmd);
+ assertEquals(new File(workerRoot), cmd.pwd);
+ },
+ "storm.home", stormHome,
+ "storm.log.dir", stormLogDir);
}
@Test
@@ -665,33 +598,112 @@ public class BasicContainerTest {
int supervisorPort = 6628;
int port = 9999;
int memOnheap = 512;
-
+
LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
-
+
Map<String, Object> superConf = new HashMap<>();
-
+
AdvancedFSOps ops = mock(AdvancedFSOps.class);
when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
-
+
LocalState ls = mock(LocalState.class);
-
- MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new HashMap<>(), ops, "profile");
-
+
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", supervisorPort, port, la, null, ls, workerId, new HashMap<>(), ops,
+ "profile");
+
assertListEquals(Arrays.asList(
- "-Xloggc:/tmp/storm/logs/gc.worker-9999-s-01-w-01-9999.log",
- "-Xms256m",
- "-Xmx512m"),
- mc.substituteChildopts("-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m -Xmx%HEAP-MEM%m", memOnheap));
-
+ "-Xloggc:/tmp/storm/logs/gc.worker-9999-s-01-w-01-9999.log",
+ "-Xms256m",
+ "-Xmx512m"),
+ mc.substituteChildopts(
+ "-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m -Xmx%HEAP-MEM%m",
+ memOnheap));
+
assertListEquals(Arrays.asList(
- "-Xloggc:/tmp/storm/logs/gc.worker-9999-s-01-w-01-9999.log",
- "-Xms256m",
- "-Xmx512m"),
- mc.substituteChildopts(Arrays.asList("-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log","-Xms256m","-Xmx%HEAP-MEM%m"), memOnheap));
-
- assertListEquals(Collections.emptyList(),
- mc.substituteChildopts(null));
+ "-Xloggc:/tmp/storm/logs/gc.worker-9999-s-01-w-01-9999.log",
+ "-Xms256m",
+ "-Xmx512m"),
+ mc.substituteChildopts(Arrays.asList(
+ "-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log", "-Xms256m",
+ "-Xmx%HEAP-MEM%m"), memOnheap));
+
+ assertListEquals(Collections.emptyList(),
+ mc.substituteChildopts(null));
+ }
+
+ private static interface Run {
+ public void run() throws Exception;
+ }
+
+ public static class CommandRun {
+ final List<String> cmd;
+ final Map<String, String> env;
+ final File pwd;
+
+ public CommandRun(List<String> cmd, Map<String, String> env, File pwd) {
+ this.cmd = cmd;
+ this.env = env;
+ this.pwd = pwd;
+ }
+ }
+
+ public static class MockBasicContainer extends BasicContainer {
+ public final List<CommandRun> profileCmds = new ArrayList<>();
+ public final List<CommandRun> workerCmds = new ArrayList<>();
+ public MockBasicContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int supervisorPort,
+ int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager,
+ LocalState localState, String workerId, Map<String, Object> topoConf, AdvancedFSOps ops,
+ String profileCmd) throws IOException {
+ super(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, localState,
+ workerId, topoConf, ops, profileCmd);
+ }
+
+ @Override
+ protected Map<String, Object> readTopoConf() throws IOException {
+ return new HashMap<>();
+ }
+
+ @Override
+ public void createNewWorkerId() {
+ super.createNewWorkerId();
+ }
+
+ @Override
+ public List<String> substituteChildopts(Object value, int memOnheap) {
+ return super.substituteChildopts(value, memOnheap);
+ }
+
+ @Override
+ protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix,
+ File targetDir) throws IOException, InterruptedException {
+ profileCmds.add(new CommandRun(command, env, targetDir));
+ return true;
+ }
+
+ @Override
+ protected void launchWorkerProcess(List<String> command, Map<String, String> env, String logPrefix,
+ ExitCodeCallback processExitCallback, File targetDir) throws IOException {
+ workerCmds.add(new CommandRun(command, env, targetDir));
+ }
+
+ @Override
+ protected String javaCmd(String cmd) {
+ //avoid system dependent things
+ return cmd;
+ }
+
+ @Override
+ protected List<String> frameworkClasspath(SimpleVersion version) {
+ //We are not really running anything so make this
+ // simple to check for
+ return Arrays.asList("FRAMEWORK_CP");
+ }
+
+ @Override
+ protected String javaLibraryPath(String stormRoot, Map<String, Object> conf) {
+ return "JLP";
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/daemon/supervisor/ContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/ContainerTest.java b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/ContainerTest.java
index c020244..b510838 100644
--- a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/ContainerTest.java
+++ b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/ContainerTest.java
@@ -1,25 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.daemon.supervisor;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+package org.apache.storm.daemon.supervisor;
+import com.google.common.base.Joiner;
import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
@@ -31,7 +24,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
import org.apache.storm.container.ResourceIsolationInterface;
@@ -41,58 +33,33 @@ import org.apache.storm.generated.ProfileRequest;
import org.junit.Test;
import org.yaml.snakeyaml.Yaml;
-import com.google.common.base.Joiner;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class ContainerTest {
- public static class MockContainer extends Container {
-
- protected MockContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int supervisorPort,
- int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager,
- String workerId, Map<String, Object> topoConf, AdvancedFSOps ops) throws IOException {
- super(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, workerId,
- topoConf, ops);
- }
-
- public final List<Long> killedPids = new ArrayList<>();
- public final List<Long> forceKilledPids = new ArrayList<>();
- public final Set<Long> allPids = new HashSet<>();
+ private static final Joiner PATH_JOIN = Joiner.on(File.separator).skipNulls();
+ private static final String DOUBLE_SEP = File.separator + File.separator;
- @Override
- protected void kill(long pid) {
- killedPids.add(pid);
- }
-
- @Override
- protected void forceKill(long pid) {
- forceKilledPids.add(pid);
- }
-
- @Override
- protected Set<Long> getAllPids() throws IOException {
- return allPids;
- }
-
- @Override
- public void launch() throws IOException {
- fail("THIS IS NOT UNDER TEST");
- }
+ static String asAbsPath(String... parts) {
+ return (File.separator + PATH_JOIN.join(parts)).replace(DOUBLE_SEP, File.separator);
+ }
- @Override
- public void relaunch() throws IOException {
- fail("THIS IS NOT UNDER TEST");
- }
+ static File asAbsFile(String... parts) {
+ return new File(asAbsPath(parts));
+ }
- @Override
- public boolean didMainProcessExit() {
- fail("THIS IS NOT UNDER TEST");
- return false;
- }
+ static String asPath(String... parts) {
+ return PATH_JOIN.join(parts);
+ }
- @Override
- public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException {
- fail("THIS IS NOT UNDER TEST");
- return false;
- }
+ public static File asFile(String... parts) {
+ return new File(asPath(parts));
}
@Test
@@ -103,46 +70,28 @@ public class ContainerTest {
when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
- MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", 6628, 8080, la, null, "worker", new HashMap<>(), ops);
+ MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", 6628, 8080, la, null, "worker", new HashMap<>(), ops);
mc.kill();
assertEquals(Collections.EMPTY_LIST, mc.killedPids);
assertEquals(Collections.EMPTY_LIST, mc.forceKilledPids);
mc.forceKill();
assertEquals(Collections.EMPTY_LIST, mc.killedPids);
assertEquals(Collections.EMPTY_LIST, mc.forceKilledPids);
-
+
long pid = 987654321;
mc.allPids.add(pid);
-
+
mc.kill();
assertEquals(mc.allPids, new HashSet<>(mc.killedPids));
assertEquals(Collections.EMPTY_LIST, mc.forceKilledPids);
mc.killedPids.clear();
-
+
mc.forceKill();
assertEquals(Collections.EMPTY_LIST, mc.killedPids);
assertEquals(mc.allPids, new HashSet<>(mc.forceKilledPids));
}
-
- private static final Joiner PATH_JOIN = Joiner.on(File.separator).skipNulls();
- private static final String DOUBLE_SEP = File.separator + File.separator;
- static String asAbsPath(String ... parts) {
- return (File.separator + PATH_JOIN.join(parts)).replace(DOUBLE_SEP, File.separator);
- }
-
- static File asAbsFile(String ... parts) {
- return new File(asAbsPath(parts));
- }
-
- static String asPath(String ... parts) {
- return PATH_JOIN.join(parts);
- }
-
- public static File asFile(String ... parts) {
- return new File(asPath(parts));
- }
-
+
@SuppressWarnings("unchecked")
@Test
public void testSetup() throws Exception {
@@ -156,11 +105,11 @@ public class ContainerTest {
final File workerUserFile = asAbsFile(stormLocal, "workers-users", workerId);
final File workerRoot = asAbsFile(stormLocal, "workers", workerId);
final File distRoot = asAbsFile(stormLocal, "supervisor", "stormdist", topoId);
-
+
final Map<String, Object> topoConf = new HashMap<>();
final List<String> topoUsers = Arrays.asList("t-user-a", "t-user-b");
final List<String> logUsers = Arrays.asList("l-user-a", "l-user-b");
-
+
final List<String> topoGroups = Arrays.asList("t-group-a", "t-group-b");
final List<String> logGroups = Arrays.asList("l-group-a", "l-group-b");
@@ -168,36 +117,36 @@ public class ContainerTest {
topoConf.put(Config.TOPOLOGY_GROUPS, topoGroups);
topoConf.put(DaemonConfig.LOGS_USERS, logUsers);
topoConf.put(Config.TOPOLOGY_USERS, topoUsers);
-
+
final Map<String, Object> superConf = new HashMap<>();
superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
-
+
final StringWriter yamlDump = new StringWriter();
-
+
AdvancedFSOps ops = mock(AdvancedFSOps.class);
when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
when(ops.fileExists(workerArtifacts)).thenReturn(true);
when(ops.fileExists(workerRoot)).thenReturn(true);
when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump);
-
+
LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
la.set_owner(user);
- MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", 6628, 8080, la, null, workerId, topoConf, ops);
-
+ MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", 6628, 8080, la, null, workerId, topoConf, ops);
+
mc.setup();
-
+
//Initial Setup
verify(ops).forceMkdir(new File(workerRoot, "pids"));
verify(ops).forceMkdir(new File(workerRoot, "tmp"));
verify(ops).forceMkdir(new File(workerRoot, "heartbeats"));
verify(ops).fileExists(workerArtifacts);
-
+
//Log file permissions
verify(ops).getWriter(logMetadataFile);
-
+
String yamlResult = yamlDump.toString();
Yaml yaml = new Yaml();
Map<String, Object> result = (Map<String, Object>) yaml.load(yamlResult);
@@ -205,22 +154,22 @@ public class ContainerTest {
assertEquals(user, result.get(Config.TOPOLOGY_SUBMITTER_USER));
HashSet<String> allowedUsers = new HashSet<>(topoUsers);
allowedUsers.addAll(logUsers);
- assertEquals(allowedUsers, new HashSet<String>((List<String>)result.get(DaemonConfig.LOGS_USERS)));
-
+ assertEquals(allowedUsers, new HashSet<String>((List<String>) result.get(DaemonConfig.LOGS_USERS)));
+
HashSet<String> allowedGroups = new HashSet<>(topoGroups);
allowedGroups.addAll(logGroups);
- assertEquals(allowedGroups, new HashSet<String>((List<String>)result.get(DaemonConfig.LOGS_GROUPS)));
-
+ assertEquals(allowedGroups, new HashSet<String>((List<String>) result.get(DaemonConfig.LOGS_GROUPS)));
+
//Save the current user to help with recovery
verify(ops).dump(workerUserFile, user);
-
+
//Create links to artifacts dir
verify(ops).createSymlink(new File(workerRoot, "artifacts"), workerArtifacts);
-
- //Create links to blobs
+
+ //Create links to blobs
verify(ops, never()).createSymlink(new File(workerRoot, "resources"), new File(distRoot, "resources"));
}
-
+
@Test
public void testCleanup() throws Exception {
final int supervisorPort = 6628;
@@ -235,38 +184,88 @@ public class ContainerTest {
final File workerUserFile = asAbsFile(stormLocal, "workers-users", workerId);
final File workerRoot = asAbsFile(stormLocal, "workers", workerId);
final File workerPidsRoot = new File(workerRoot, "pids");
-
+
final Map<String, Object> topoConf = new HashMap<>();
-
+
final Map<String, Object> superConf = new HashMap<>();
superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
-
+
final StringWriter yamlDump = new StringWriter();
-
+
AdvancedFSOps ops = mock(AdvancedFSOps.class);
when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
when(ops.fileExists(workerArtifacts)).thenReturn(true);
when(ops.fileExists(workerRoot)).thenReturn(true);
when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump);
-
+
ResourceIsolationInterface iso = mock(ResourceIsolationInterface.class);
-
+
LocalAssignment la = new LocalAssignment();
la.set_owner(user);
la.set_topology_id(topoId);
- MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", supervisorPort, port, la, iso, workerId, topoConf, ops);
+ MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", supervisorPort, port, la, iso, workerId, topoConf, ops);
mc.allPids.add(pid);
-
+
mc.cleanUp();
verify(ops).deleteIfExists(eq(new File(workerPidsRoot, String.valueOf(pid))), eq(user), any(String.class));
verify(iso).releaseResourcesForWorker(workerId);
-
+
verify(ops).deleteIfExists(eq(new File(workerRoot, "pids")), eq(user), any(String.class));
verify(ops).deleteIfExists(eq(new File(workerRoot, "tmp")), eq(user), any(String.class));
verify(ops).deleteIfExists(eq(new File(workerRoot, "heartbeats")), eq(user), any(String.class));
verify(ops).deleteIfExists(eq(workerRoot), eq(user), any(String.class));
verify(ops).deleteIfExists(workerUserFile);
}
+
+ public static class MockContainer extends Container {
+
+ public final List<Long> killedPids = new ArrayList<>();
+ public final List<Long> forceKilledPids = new ArrayList<>();
+ public final Set<Long> allPids = new HashSet<>();
+ protected MockContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int supervisorPort,
+ int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager,
+ String workerId, Map<String, Object> topoConf, AdvancedFSOps ops) throws IOException {
+ super(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, workerId,
+ topoConf, ops);
+ }
+
+ @Override
+ protected void kill(long pid) {
+ killedPids.add(pid);
+ }
+
+ @Override
+ protected void forceKill(long pid) {
+ forceKilledPids.add(pid);
+ }
+
+ @Override
+ protected Set<Long> getAllPids() throws IOException {
+ return allPids;
+ }
+
+ @Override
+ public void launch() throws IOException {
+ fail("THIS IS NOT UNDER TEST");
+ }
+
+ @Override
+ public void relaunch() throws IOException {
+ fail("THIS IS NOT UNDER TEST");
+ }
+
+ @Override
+ public boolean didMainProcessExit() {
+ fail("THIS IS NOT UNDER TEST");
+ return false;
+ }
+
+ @Override
+ public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException {
+ fail("THIS IS NOT UNDER TEST");
+ return false;
+ }
+ }
}
[47/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-kafka
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionManager.java
index a9a18fd..5805c21 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionManager.java
@@ -1,26 +1,30 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import java.io.Serializable;
-
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import kafka.javaapi.consumer.SimpleConsumer;
+import kafka.javaapi.message.ByteBufferMessageSet;
+import kafka.message.MessageAndOffset;
import org.apache.storm.Config;
import org.apache.storm.kafka.KafkaSpout.EmitState;
import org.apache.storm.kafka.trident.MaxMetric;
@@ -32,12 +36,6 @@ import org.apache.storm.spout.SpoutOutputCollector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
-
-import kafka.javaapi.consumer.SimpleConsumer;
-import kafka.javaapi.message.ByteBufferMessageSet;
-import kafka.message.MessageAndOffset;
-
public class PartitionManager {
private static final Logger LOG = LoggerFactory.getLogger(PartitionManager.class);
@@ -50,11 +48,8 @@ public class PartitionManager {
// Count of messages which were not retried because failedMsgRetryManager didn't consider offset eligible for
// retry
private final CountMetric _messageIneligibleForRetryCount;
- Long _emittedToOffset;
- // _pending key = Kafka offset, value = time at which the message was first submitted to the topology
- private SortedMap<Long,Long> _pending = new TreeMap<Long,Long>();
private final FailedMsgRetryManager _failedMsgRetryManager;
-
+ Long _emittedToOffset;
// retryRecords key = Kafka offset, value = retry info for the given message
Long _committedTo;
LinkedList<MessageAndOffset> _waitingToEmit = new LinkedList<MessageAndOffset>();
@@ -66,15 +61,16 @@ public class PartitionManager {
ZkState _state;
Map _topoConf;
long numberFailed, numberAcked;
+ // _pending key = Kafka offset, value = time at which the message was first submitted to the topology
+ private SortedMap<Long, Long> _pending = new TreeMap<Long, Long>();
public PartitionManager(
- DynamicPartitionConnections connections,
- String topologyInstanceId,
- ZkState state,
- Map<String, Object> topoConf,
- SpoutConfig spoutConfig,
- Partition id)
- {
+ DynamicPartitionConnections connections,
+ String topologyInstanceId,
+ ZkState state,
+ Map<String, Object> topoConf,
+ SpoutConfig spoutConfig,
+ Partition id) {
this(connections, topologyInstanceId, state, topoConf, spoutConfig, id, null);
}
@@ -82,13 +78,13 @@ public class PartitionManager {
* @param previousManager previous partition manager if manager for partition is being recreated
*/
public PartitionManager(
- DynamicPartitionConnections connections,
- String topologyInstanceId,
- ZkState state,
- Map<String, Object> topoConf,
- SpoutConfig spoutConfig,
- Partition id,
- PartitionManager previousManager) {
+ DynamicPartitionConnections connections,
+ String topologyInstanceId,
+ ZkState state,
+ Map<String, Object> topoConf,
+ SpoutConfig spoutConfig,
+ Partition id,
+ PartitionManager previousManager) {
_partition = id;
_connections = connections;
_spoutConfig = spoutConfig;
@@ -105,16 +101,16 @@ public class PartitionManager {
_waitingToEmit = previousManager._waitingToEmit;
_pending = previousManager._pending;
LOG.info("Recreating PartitionManager based on previous manager, _waitingToEmit size: {}, _pending size: {}",
- _waitingToEmit.size(),
- _pending.size());
+ _waitingToEmit.size(),
+ _pending.size());
} else {
try {
_failedMsgRetryManager = (FailedMsgRetryManager) Class.forName(spoutConfig.failedMsgRetryManagerClass).newInstance();
_failedMsgRetryManager.prepare(spoutConfig, _topoConf);
} catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) {
throw new IllegalArgumentException(String.format("Failed to create an instance of <%s> from: <%s>",
- FailedMsgRetryManager.class,
- spoutConfig.failedMsgRetryManagerClass), e);
+ FailedMsgRetryManager.class,
+ spoutConfig.failedMsgRetryManagerClass), e);
}
String jsonTopologyId = null;
@@ -142,7 +138,8 @@ public class PartitionManager {
LOG.info("Topology change detected and ignore zookeeper offsets set to true, using configuration to determine offset");
} else {
_committedTo = jsonOffset;
- LOG.info("Read last commit offset from zookeeper: " + _committedTo + "; old topology_id: " + jsonTopologyId + " - new topology_id: " + topologyInstanceId);
+ LOG.info("Read last commit offset from zookeeper: " + _committedTo + "; old topology_id: " + jsonTopologyId +
+ " - new topology_id: " + topologyInstanceId);
}
if (currentOffset - _committedTo > spoutConfig.maxOffsetBehind || _committedTo <= 0) {
@@ -150,7 +147,8 @@ public class PartitionManager {
Long lastCommittedOffset = _committedTo;
_committedTo = currentOffset;
LOG.info("Commit offset " + lastCommittedOffset + " is more than " +
- spoutConfig.maxOffsetBehind + " behind latest offset " + currentOffset + ", resetting to startOffsetTime=" + spoutConfig.startOffsetTime);
+ spoutConfig.maxOffsetBehind + " behind latest offset " + currentOffset + ", resetting to startOffsetTime=" +
+ spoutConfig.startOffsetTime);
}
LOG.info("Starting Kafka " + _consumer.host() + " " + id + " from offset " + _committedTo);
@@ -191,13 +189,14 @@ public class PartitionManager {
Iterable<List<Object>> tups;
if (_spoutConfig.scheme instanceof MessageMetadataSchemeAsMultiScheme) {
- tups = KafkaUtils.generateTuples((MessageMetadataSchemeAsMultiScheme) _spoutConfig.scheme, toEmit.message(), _partition, toEmit.offset());
+ tups = KafkaUtils.generateTuples((MessageMetadataSchemeAsMultiScheme) _spoutConfig.scheme, toEmit.message(), _partition,
+ toEmit.offset());
} else {
tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.message(), _partition.topic);
}
if ((tups != null) && tups.iterator().hasNext()) {
- if (!Strings.isNullOrEmpty(_spoutConfig.outputStreamId)) {
+ if (!Strings.isNullOrEmpty(_spoutConfig.outputStreamId)) {
for (List<Object> tup : tups) {
collector.emit(_spoutConfig.outputStreamId, tup, new KafkaMessageId(_partition, toEmit.offset()));
}
@@ -305,16 +304,17 @@ public class PartitionManager {
public void fail(Long offset) {
if (offset < _emittedToOffset - _spoutConfig.maxOffsetBehind) {
LOG.info(
- "Skipping failed tuple at offset={}" +
- " because it's more than maxOffsetBehind={}" +
- " behind _emittedToOffset={} for {}",
+ "Skipping failed tuple at offset={}" +
+ " because it's more than maxOffsetBehind={}" +
+ " behind _emittedToOffset={} for {}",
offset,
_spoutConfig.maxOffsetBehind,
_emittedToOffset,
_partition
);
} else {
- LOG.debug("Failing at offset={} with _pending.size()={} pending and _emittedToOffset={} for {}", offset, _pending.size(), _emittedToOffset, _partition);
+ LOG.debug("Failing at offset={} with _pending.size()={} pending and _emittedToOffset={} for {}", offset, _pending.size(),
+ _emittedToOffset, _partition);
numberFailed++;
if (numberAcked == 0 && numberFailed > _spoutConfig.maxOffsetBehind) {
throw new RuntimeException("Too many tuple failures");
@@ -337,19 +337,22 @@ public class PartitionManager {
public void commit() {
long lastCompletedOffset = lastCompletedOffset();
if (_committedTo != lastCompletedOffset) {
- LOG.debug("Writing last completed offset ({}) to ZK for {} for topology: {}", lastCompletedOffset, _partition, _topologyInstanceId);
+ LOG.debug("Writing last completed offset ({}) to ZK for {} for topology: {}", lastCompletedOffset, _partition,
+ _topologyInstanceId);
Map<Object, Object> data = (Map<Object, Object>) ImmutableMap.builder()
- .put("topology", ImmutableMap.of("id", _topologyInstanceId,
- "name", _topoConf.get(Config.TOPOLOGY_NAME)))
- .put("offset", lastCompletedOffset)
- .put("partition", _partition.partition)
- .put("broker", ImmutableMap.of("host", _partition.host.host,
- "port", _partition.host.port))
- .put("topic", _partition.topic).build();
+ .put("topology", ImmutableMap.of("id", _topologyInstanceId,
+ "name", _topoConf
+ .get(Config.TOPOLOGY_NAME)))
+ .put("offset", lastCompletedOffset)
+ .put("partition", _partition.partition)
+ .put("broker", ImmutableMap.of("host", _partition.host.host,
+ "port", _partition.host.port))
+ .put("topic", _partition.topic).build();
_state.writeJSON(committedPath(), data);
_committedTo = lastCompletedOffset;
- LOG.debug("Wrote last completed offset ({}) to ZK for {} for topology: {}", lastCompletedOffset, _partition, _topologyInstanceId);
+ LOG.debug("Wrote last completed offset ({}) to ZK for {} for topology: {}", lastCompletedOffset, _partition,
+ _topologyInstanceId);
} else {
LOG.debug("No new offset for {} for topology: {}", _partition, _topologyInstanceId);
}
@@ -377,7 +380,7 @@ public class PartitionManager {
public void close() {
commit();
- _connections.unregister(_partition.host, _partition.topic , _partition.partition);
+ _connections.unregister(_partition.host, _partition.topic, _partition.partition);
}
static class KafkaMessageId implements Serializable {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/SpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/SpoutConfig.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/SpoutConfig.java
index 0ad19d7..74a4a3b 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/SpoutConfig.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/SpoutConfig.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticCoordinator.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticCoordinator.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticCoordinator.java
index cd23ca6..8d12ee1 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticCoordinator.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticCoordinator.java
@@ -1,33 +1,30 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.kafka.trident.GlobalPartitionInformation;
-import java.util.*;
-
public class StaticCoordinator implements PartitionCoordinator {
Map<Partition, PartitionManager> _managers = new HashMap<Partition, PartitionManager>();
List<PartitionManager> _allManagers = new ArrayList<>();
public StaticCoordinator(DynamicPartitionConnections connections, Map<String, Object> topoConf, SpoutConfig config, ZkState state,
- int taskIndex, int totalTasks, int taskId, String topologyInstanceId) {
+ int taskIndex, int totalTasks, int taskId, String topologyInstanceId) {
StaticHosts hosts = (StaticHosts) config.hosts;
List<GlobalPartitionInformation> partitions = new ArrayList<GlobalPartitionInformation>();
partitions.add(hosts.getPartitionInformation());
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticHosts.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticHosts.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticHosts.java
index 33d5c16..1f8f903 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticHosts.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticHosts.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import org.apache.storm.kafka.trident.GlobalPartitionInformation;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticPartitionConnections.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticPartitionConnections.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticPartitionConnections.java
index 77a7211..2c9d4f2 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticPartitionConnections.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StaticPartitionConnections.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka;
-import kafka.javaapi.consumer.SimpleConsumer;
+package org.apache.storm.kafka;
import java.util.HashMap;
import java.util.Map;
+import kafka.javaapi.consumer.SimpleConsumer;
public class StaticPartitionConnections {
Map<Integer, SimpleConsumer> _kafka = new HashMap<Integer, SimpleConsumer>();
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringKeyValueScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringKeyValueScheme.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringKeyValueScheme.java
index 9ef7f74..3d62961 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringKeyValueScheme.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringKeyValueScheme.java
@@ -1,33 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
-import org.apache.storm.tuple.Values;
import com.google.common.collect.ImmutableMap;
-
import java.nio.ByteBuffer;
import java.util.List;
+import org.apache.storm.tuple.Values;
public class StringKeyValueScheme extends StringScheme implements KeyValueScheme {
@Override
public List<Object> deserializeKeyAndValue(ByteBuffer key, ByteBuffer value) {
- if ( key == null ) {
+ if (key == null) {
return deserialize(value);
}
String keyString = StringScheme.deserializeString(key);
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringMessageAndMetadataScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringMessageAndMetadataScheme.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringMessageAndMetadataScheme.java
index e57738d..ab6e500 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringMessageAndMetadataScheme.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringMessageAndMetadataScheme.java
@@ -1,33 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Values;
+package org.apache.storm.kafka;
import java.nio.ByteBuffer;
import java.util.List;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Values;
public class StringMessageAndMetadataScheme extends StringScheme implements MessageMetadataScheme {
- private static final long serialVersionUID = -5441841920447947374L;
-
public static final String STRING_SCHEME_PARTITION_KEY = "partition";
public static final String STRING_SCHEME_OFFSET = "offset";
+ private static final long serialVersionUID = -5441841920447947374L;
@Override
public List<Object> deserializeMessageWithMetadata(ByteBuffer message, Partition partition, long offset) {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringMultiSchemeWithTopic.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringMultiSchemeWithTopic.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringMultiSchemeWithTopic.java
index e197318..061b30a 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringMultiSchemeWithTopic.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringMultiSchemeWithTopic.java
@@ -1,32 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka;
-import org.apache.storm.spout.MultiScheme;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Values;
+package org.apache.storm.kafka;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
+import org.apache.storm.spout.MultiScheme;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Values;
public class StringMultiSchemeWithTopic
- implements MultiScheme {
+ implements MultiScheme {
public static final String STRING_SCHEME_KEY = "str";
public static final String TOPIC_KEY = "topic";
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringScheme.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringScheme.java
index b0b8d27..bcbc058 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringScheme.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/StringScheme.java
@@ -1,39 +1,29 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka;
-import org.apache.storm.spout.Scheme;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Values;
-import org.apache.storm.utils.Utils;
+package org.apache.storm.kafka;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.List;
+import org.apache.storm.spout.Scheme;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Values;
+import org.apache.storm.utils.Utils;
public class StringScheme implements Scheme {
- private static final Charset UTF8_CHARSET = StandardCharsets.UTF_8;
public static final String STRING_SCHEME_KEY = "str";
-
- public List<Object> deserialize(ByteBuffer bytes) {
- return new Values(deserializeString(bytes));
- }
+ private static final Charset UTF8_CHARSET = StandardCharsets.UTF_8;
public static String deserializeString(ByteBuffer string) {
if (string.hasArray()) {
@@ -44,6 +34,10 @@ public class StringScheme implements Scheme {
}
}
+ public List<Object> deserialize(ByteBuffer bytes) {
+ return new Values(deserializeString(bytes));
+ }
+
public Fields getOutputFields() {
return new Fields(STRING_SCHEME_KEY);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/TopicOffsetOutOfRangeException.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/TopicOffsetOutOfRangeException.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/TopicOffsetOutOfRangeException.java
index 8e1c98f..613a62e 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/TopicOffsetOutOfRangeException.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/TopicOffsetOutOfRangeException.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.kafka;
public class TopicOffsetOutOfRangeException extends RuntimeException {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkCoordinator.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkCoordinator.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkCoordinator.java
index 136dc51..bc9ebd5 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkCoordinator.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkCoordinator.java
@@ -1,27 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.storm.kafka.trident.GlobalPartitionInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.storm.kafka.trident.GlobalPartitionInformation;
-
-import java.util.*;
import static org.apache.storm.kafka.KafkaUtils.taskPrefix;
@@ -43,12 +42,13 @@ public class ZkCoordinator implements PartitionCoordinator {
Map _topoConf;
public ZkCoordinator(DynamicPartitionConnections connections, Map<String, Object> topoConf, SpoutConfig spoutConfig, ZkState state,
- int taskIndex, int totalTasks, int taskId, String topologyInstanceId) {
- this(connections, topoConf, spoutConfig, state, taskIndex, totalTasks, taskId, topologyInstanceId, buildReader(topoConf, spoutConfig));
+ int taskIndex, int totalTasks, int taskId, String topologyInstanceId) {
+ this(connections, topoConf, spoutConfig, state, taskIndex, totalTasks, taskId, topologyInstanceId,
+ buildReader(topoConf, spoutConfig));
}
public ZkCoordinator(DynamicPartitionConnections connections, Map<String, Object> topoConf, SpoutConfig spoutConfig, ZkState state,
- int taskIndex, int totalTasks, int taskId, String topologyInstanceId, DynamicBrokersReader reader) {
+ int taskIndex, int totalTasks, int taskId, String topologyInstanceId, DynamicBrokersReader reader) {
_spoutConfig = spoutConfig;
_connections = connections;
_taskIndex = taskIndex;
@@ -103,13 +103,13 @@ public class ZkCoordinator implements PartitionCoordinator {
for (Partition id : newPartitions) {
PartitionManager man = new PartitionManager(
- _connections,
- _topologyInstanceId,
- _state,
- _topoConf,
- _spoutConfig,
- id,
- deletedManagers.get(id.partition));
+ _connections,
+ _topologyInstanceId,
+ _state,
+ _topoConf,
+ _spoutConfig,
+ id,
+ deletedManagers.get(id.partition));
_managers.put(id, man);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkHosts.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkHosts.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkHosts.java
index 2c2a26f..9c6b29d 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkHosts.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkHosts.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkState.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkState.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkState.java
index 7241c60..3d27173 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkState.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ZkState.java
@@ -1,53 +1,59 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
-import org.apache.storm.Config;
-import org.apache.storm.utils.ObjectReader;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryNTimes;
+import org.apache.storm.Config;
+import org.apache.storm.utils.ObjectReader;
import org.apache.zookeeper.CreateMode;
import org.json.simple.JSONValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
public class ZkState {
private static final Logger LOG = LoggerFactory.getLogger(ZkState.class);
CuratorFramework _curator;
+ public ZkState(Map<String, Object> stateConf) {
+ stateConf = new HashMap<>(stateConf);
+
+ try {
+ _curator = newCurator(stateConf);
+ _curator.start();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
private CuratorFramework newCurator(final Map<String, Object> stateConf)
- throws Exception {
+ throws Exception {
Integer port = (Integer) stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_PORT);
String serverPorts = "";
for (String server : (List<String>) stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS)) {
serverPorts = serverPorts + server + ":" + port + ",";
}
return CuratorFrameworkFactory.newClient(serverPorts,
- ObjectReader.getInt(stateConf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)),
- ObjectReader.getInt(stateConf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT)),
- new RetryNTimes(ObjectReader.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)),
- ObjectReader.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL))));
+ ObjectReader.getInt(stateConf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)),
+ ObjectReader.getInt(stateConf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT)),
+ new RetryNTimes(ObjectReader.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)),
+ ObjectReader
+ .getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL))));
}
public CuratorFramework getCurator() {
@@ -55,17 +61,6 @@ public class ZkState {
return _curator;
}
- public ZkState(Map<String, Object> stateConf) {
- stateConf = new HashMap<>(stateConf);
-
- try {
- _curator = newCurator(stateConf);
- _curator.start();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
public void writeJSON(String path, Map<Object, Object> data) {
LOG.debug("Writing {} the data {}", path, data.toString());
writeBytes(path, JSONValue.toJSONString(data).getBytes(Charset.forName("UTF-8")));
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/KafkaBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/KafkaBolt.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/KafkaBolt.java
index a4c255a..e0b94f3 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/KafkaBolt.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/KafkaBolt.java
@@ -1,43 +1,36 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.bolt;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.topology.base.BaseRichBolt;
-import org.apache.storm.topology.base.BaseTickTupleAwareRichBolt;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.utils.TupleUtils;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
-import org.apache.kafka.clients.producer.Callback;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.storm.kafka.bolt.mapper.FieldNameBasedTupleToKafkaMapper;
import org.apache.storm.kafka.bolt.mapper.TupleToKafkaMapper;
import org.apache.storm.kafka.bolt.selector.DefaultTopicSelector;
import org.apache.storm.kafka.bolt.selector.KafkaTopicSelector;
-import java.util.concurrent.Future;
-import java.util.concurrent.ExecutionException;
-import java.util.Map;
-import java.util.Properties;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseTickTupleAwareRichBolt;
+import org.apache.storm.tuple.Tuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -57,13 +50,11 @@ import java.util.Properties;
@Deprecated
public class KafkaBolt<K, V> extends BaseTickTupleAwareRichBolt {
- private static final Logger LOG = LoggerFactory.getLogger(KafkaBolt.class);
-
public static final String TOPIC = "topic";
-
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaBolt.class);
private KafkaProducer<K, V> producer;
private OutputCollector collector;
- private TupleToKafkaMapper<K,V> mapper;
+ private TupleToKafkaMapper<K, V> mapper;
private KafkaTopicSelector topicSelector;
private Properties boltSpecfiedProperties = new Properties();
/**
@@ -76,18 +67,18 @@ public class KafkaBolt<K, V> extends BaseTickTupleAwareRichBolt {
private boolean async = true;
public KafkaBolt() {}
-
- public KafkaBolt<K,V> withTupleToKafkaMapper(TupleToKafkaMapper<K,V> mapper) {
+
+ public KafkaBolt<K, V> withTupleToKafkaMapper(TupleToKafkaMapper<K, V> mapper) {
this.mapper = mapper;
return this;
}
- public KafkaBolt<K,V> withTopicSelector(KafkaTopicSelector selector) {
+ public KafkaBolt<K, V> withTopicSelector(KafkaTopicSelector selector) {
this.topicSelector = selector;
return this;
}
- public KafkaBolt<K,V> withProducerProperties(Properties producerProperties) {
+ public KafkaBolt<K, V> withProducerProperties(Properties producerProperties) {
this.boltSpecfiedProperties = producerProperties;
return this;
}
@@ -95,13 +86,13 @@ public class KafkaBolt<K, V> extends BaseTickTupleAwareRichBolt {
@Override
public void prepare(Map<String, Object> topoConf, TopologyContext context, OutputCollector collector) {
//for backward compatibility.
- if(mapper == null) {
- this.mapper = new FieldNameBasedTupleToKafkaMapper<K,V>();
+ if (mapper == null) {
+ this.mapper = new FieldNameBasedTupleToKafkaMapper<K, V>();
}
//for backward compatibility.
- if(topicSelector == null) {
- if(topoConf.containsKey(TOPIC)) {
+ if (topicSelector == null) {
+ if (topoConf.containsKey(TOPIC)) {
this.topicSelector = new DefaultTopicSelector((String) topoConf.get(TOPIC));
} else {
throw new IllegalArgumentException("topic should be specified in bolt's configuration");
@@ -121,7 +112,7 @@ public class KafkaBolt<K, V> extends BaseTickTupleAwareRichBolt {
key = mapper.getKeyFromTuple(input);
message = mapper.getMessageFromTuple(input);
topic = topicSelector.getTopic(input);
- if (topic != null ) {
+ if (topic != null) {
Callback callback = null;
if (!fireAndForget && async) {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/mapper/FieldNameBasedTupleToKafkaMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/mapper/FieldNameBasedTupleToKafkaMapper.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/mapper/FieldNameBasedTupleToKafkaMapper.java
index 672da8e..7276ef6 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/mapper/FieldNameBasedTupleToKafkaMapper.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/mapper/FieldNameBasedTupleToKafkaMapper.java
@@ -1,25 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.bolt.mapper;
import org.apache.storm.tuple.Tuple;
-public class FieldNameBasedTupleToKafkaMapper<K,V> implements TupleToKafkaMapper<K, V> {
+public class FieldNameBasedTupleToKafkaMapper<K, V> implements TupleToKafkaMapper<K, V> {
public static final String BOLT_KEY = "key";
public static final String BOLT_MESSAGE = "message";
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/mapper/TupleToKafkaMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/mapper/TupleToKafkaMapper.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/mapper/TupleToKafkaMapper.java
index 3890413..7012e6b 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/mapper/TupleToKafkaMapper.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/mapper/TupleToKafkaMapper.java
@@ -1,32 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka.bolt.mapper;
-import org.apache.storm.tuple.Tuple;
+package org.apache.storm.kafka.bolt.mapper;
import java.io.Serializable;
+import org.apache.storm.tuple.Tuple;
/**
* as the really verbose name suggests this interface mapps a storm tuple to kafka key and message.
* @param <K> type of key.
* @param <V> type of value.
*/
-public interface TupleToKafkaMapper<K,V> extends Serializable {
+public interface TupleToKafkaMapper<K, V> extends Serializable {
K getKeyFromTuple(Tuple tuple);
+
V getMessageFromTuple(Tuple tuple);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/DefaultTopicSelector.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/DefaultTopicSelector.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/DefaultTopicSelector.java
index 2aafc78..d1784b0 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/DefaultTopicSelector.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/DefaultTopicSelector.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.bolt.selector;
import org.apache.storm.tuple.Tuple;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/FieldIndexTopicSelector.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/FieldIndexTopicSelector.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/FieldIndexTopicSelector.java
index 7b52403..50c5c1f 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/FieldIndexTopicSelector.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/FieldIndexTopicSelector.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.bolt.selector;
import org.apache.storm.tuple.Tuple;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/FieldNameTopicSelector.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/FieldNameTopicSelector.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/FieldNameTopicSelector.java
index a622e8f..d3c304a 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/FieldNameTopicSelector.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/FieldNameTopicSelector.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.bolt.selector;
import org.apache.storm.tuple.Tuple;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/KafkaTopicSelector.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/KafkaTopicSelector.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/KafkaTopicSelector.java
index cb7fb44..4045df7 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/KafkaTopicSelector.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/bolt/selector/KafkaTopicSelector.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka.bolt.selector;
-import org.apache.storm.tuple.Tuple;
+package org.apache.storm.kafka.bolt.selector;
import java.io.Serializable;
+import org.apache.storm.tuple.Tuple;
public interface KafkaTopicSelector extends Serializable {
String getTopic(Tuple tuple);
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/Coordinator.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/Coordinator.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/Coordinator.java
index 76baf62..0e8dba1 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/Coordinator.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/Coordinator.java
@@ -1,30 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.kafka.KafkaUtils;
import org.apache.storm.trident.spout.IOpaquePartitionedTridentSpout;
import org.apache.storm.trident.spout.IPartitionedTridentSpout;
-import java.util.List;
-import java.util.Map;
-
-class Coordinator implements IPartitionedTridentSpout.Coordinator<List<GlobalPartitionInformation>>, IOpaquePartitionedTridentSpout.Coordinator<List<GlobalPartitionInformation>> {
+class Coordinator implements IPartitionedTridentSpout.Coordinator<List<GlobalPartitionInformation>>,
+ IOpaquePartitionedTridentSpout.Coordinator<List<GlobalPartitionInformation>> {
private IBrokerReader reader;
private TridentKafkaConfig config;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/DefaultCoordinator.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/DefaultCoordinator.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/DefaultCoordinator.java
index 7a7e32c..575e235 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/DefaultCoordinator.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/DefaultCoordinator.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
public class DefaultCoordinator implements IBatchCoordinator {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/GlobalPartitionInformation.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/GlobalPartitionInformation.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/GlobalPartitionInformation.java
index e420cb3..b26dc7f 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/GlobalPartitionInformation.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/GlobalPartitionInformation.java
@@ -1,35 +1,32 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
import com.google.common.base.Objects;
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
import org.apache.storm.kafka.Broker;
import org.apache.storm.kafka.Partition;
-import java.io.Serializable;
-import java.util.*;
-
public class GlobalPartitionInformation implements Iterable<Partition>, Serializable {
- private Map<Integer, Broker> partitionMap;
public String topic;
-
+ private Map<Integer, Broker> partitionMap;
//Flag to keep the Partition Path Id backward compatible with Old implementation of Partition.getId() == "partition_" + partition
private Boolean bUseTopicNameForPartitionPathId;
@@ -49,16 +46,16 @@ public class GlobalPartitionInformation implements Iterable<Partition>, Serializ
partitionMap.put(partitionId, broker);
}
- public Boolean getbUseTopicNameForPartitionPathId () {
+ public Boolean getbUseTopicNameForPartitionPathId() {
return bUseTopicNameForPartitionPathId;
}
@Override
public String toString() {
return "GlobalPartitionInformation{" +
- "topic=" + topic +
- ", partitionMap=" + partitionMap +
- '}';
+ "topic=" + topic +
+ ", partitionMap=" + partitionMap +
+ '}';
}
public Broker getBrokerFor(Integer partitionId) {
@@ -87,7 +84,7 @@ public class GlobalPartitionInformation implements Iterable<Partition>, Serializ
@Override
public Partition next() {
Map.Entry<Integer, Broker> next = iterator.next();
- return new Partition(next.getValue(), topic , next.getKey(), bUseTopicNameForPartitionPathId);
+ return new Partition(next.getValue(), topic, next.getKey(), bUseTopicNameForPartitionPathId);
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/IBatchCoordinator.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/IBatchCoordinator.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/IBatchCoordinator.java
index 41369ba..4c6c404 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/IBatchCoordinator.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/IBatchCoordinator.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/IBrokerReader.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/IBrokerReader.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/IBrokerReader.java
index 904d8c9..c5cf8b2 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/IBrokerReader.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/IBrokerReader.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
import java.util.List;
-import java.util.Map;
public interface IBrokerReader {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/MaxMetric.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/MaxMetric.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/MaxMetric.java
index 2332205..14324ed 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/MaxMetric.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/MaxMetric.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
[25/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-eventhubs
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-eventhubs
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/18723171
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/18723171
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/18723171
Branch: refs/heads/master
Commit: 18723171612bdfe818929297378433a3c069e4e7
Parents: fc1cf09
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 21:35:57 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 00:22:17 2018 -0400
----------------------------------------------------------------------
external/storm-eventhubs/pom.xml | 2 +-
.../eventhubs/bolt/DefaultEventDataFormat.java | 38 +-
.../storm/eventhubs/bolt/EventHubBolt.java | 208 +++++----
.../eventhubs/bolt/EventHubBoltConfig.java | 147 +++---
.../storm/eventhubs/bolt/IEventDataFormat.java | 3 +-
.../eventhubs/spout/BinaryEventDataScheme.java | 78 ++--
.../storm/eventhubs/spout/EventDataScheme.java | 92 ++--
.../storm/eventhubs/spout/EventDataWrap.java | 51 ++-
.../storm/eventhubs/spout/EventHubFilter.java | 11 +-
.../eventhubs/spout/EventHubReceiverImpl.java | 237 +++++-----
.../storm/eventhubs/spout/EventHubSpout.java | 420 ++++++++---------
.../eventhubs/spout/EventHubSpoutConfig.java | 451 +++++++++----------
.../storm/eventhubs/spout/FieldConstants.java | 13 +-
.../storm/eventhubs/spout/IEventDataScheme.java | 34 +-
.../eventhubs/spout/IEventHubReceiver.java | 11 +-
.../spout/IEventHubReceiverFactory.java | 27 +-
.../eventhubs/spout/IPartitionCoordinator.java | 5 +-
.../eventhubs/spout/IPartitionManager.java | 17 +-
.../spout/IPartitionManagerFactory.java | 33 +-
.../storm/eventhubs/spout/IStateStore.java | 9 +-
.../apache/storm/eventhubs/spout/MessageId.java | 69 +--
.../storm/eventhubs/spout/PartitionManager.java | 126 +++---
.../eventhubs/spout/SimplePartitionManager.java | 200 ++++----
.../spout/StaticPartitionCoordinator.java | 92 ++--
.../eventhubs/spout/StringEventDataScheme.java | 78 ++--
.../eventhubs/spout/ZookeeperStateStore.java | 107 ++---
.../storm/eventhubs/trident/Coordinator.java | 65 ++-
.../trident/ITridentPartitionManager.java | 26 +-
.../ITridentPartitionManagerFactory.java | 4 +-
.../trident/OpaqueTridentEventHubEmitter.java | 69 +--
.../trident/OpaqueTridentEventHubSpout.java | 65 ++-
.../storm/eventhubs/trident/Partition.java | 23 +-
.../storm/eventhubs/trident/Partitions.java | 28 +-
.../TransactionalTridentEventHubEmitter.java | 240 +++++-----
.../TransactionalTridentEventHubSpout.java | 64 ++-
.../trident/TridentPartitionManager.java | 117 ++---
.../eventhubs/samples/AtMostOnceEventCount.java | 44 +-
.../storm/eventhubs/samples/EventCount.java | 208 +++++----
.../storm/eventhubs/samples/EventHubLoop.java | 44 +-
.../samples/OpaqueTridentEventCount.java | 45 +-
.../samples/TransactionalTridentEventCount.java | 86 ++--
.../eventhubs/samples/bolt/GlobalCountBolt.java | 95 ++--
.../samples/bolt/PartialCountBolt.java | 59 ++-
.../eventhubs/spout/EventHubReceiverMock.java | 97 ++--
.../spout/EventHubSpoutCallerMock.java | 130 +++---
.../spout/PartitionManagerCallerMock.java | 134 +++---
.../spout/SpoutOutputCollectorMock.java | 76 ++--
.../storm/eventhubs/spout/StateStoreMock.java | 44 +-
.../storm/eventhubs/spout/TestEventData.java | 29 +-
.../eventhubs/spout/TestEventHubSpout.java | 82 ++--
.../eventhubs/spout/TestPartitionManager.java | 178 ++++----
.../TestTransactionalTridentEmitter.java | 123 +++--
.../eventhubs/trident/TridentCollectorMock.java | 58 +--
53 files changed, 2386 insertions(+), 2406 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/pom.xml b/external/storm-eventhubs/pom.xml
index 79b16a9..0c7d3a8 100755
--- a/external/storm-eventhubs/pom.xml
+++ b/external/storm-eventhubs/pom.xml
@@ -53,7 +53,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>1765</maxAllowedViolations>
+ <maxAllowedViolations>45</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/DefaultEventDataFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/DefaultEventDataFormat.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/DefaultEventDataFormat.java
index d6e1dbc..094ab9a 100644
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/DefaultEventDataFormat.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/DefaultEventDataFormat.java
@@ -15,33 +15,33 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.bolt;
import org.apache.storm.tuple.Tuple;
/**
- * A default implementation of IEventDataFormat that converts the tuple
- * into a delimited string.
+ * A default implementation of IEventDataFormat that converts the tuple into a delimited string.
*/
public class DefaultEventDataFormat implements IEventDataFormat {
- private static final long serialVersionUID = 1L;
- private String delimiter = ",";
-
- public DefaultEventDataFormat withFieldDelimiter(String delimiter) {
- this.delimiter = delimiter;
- return this;
- }
+ private static final long serialVersionUID = 1L;
+ private String delimiter = ",";
+
+ public DefaultEventDataFormat withFieldDelimiter(String delimiter) {
+ this.delimiter = delimiter;
+ return this;
+ }
- @Override
- public byte[] serialize(Tuple tuple) {
- StringBuilder sb = new StringBuilder();
- for(Object obj : tuple.getValues()) {
- if(sb.length() != 0) {
- sb.append(delimiter);
- }
- sb.append(obj.toString());
+ @Override
+ public byte[] serialize(Tuple tuple) {
+ StringBuilder sb = new StringBuilder();
+ for (Object obj : tuple.getValues()) {
+ if (sb.length() != 0) {
+ sb.append(delimiter);
+ }
+ sb.append(obj.toString());
+ }
+ return sb.toString().getBytes();
}
- return sb.toString().getBytes();
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java
index 7d1aeab..85ffd03 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBolt.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.bolt;
@@ -22,6 +23,8 @@ import com.microsoft.azure.eventhubs.EventData;
import com.microsoft.azure.eventhubs.EventHubClient;
import com.microsoft.azure.eventhubs.PartitionSender;
import com.microsoft.azure.servicebus.ServiceBusException;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
import org.apache.storm.eventhubs.spout.EventHubException;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -31,116 +34,107 @@ import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-
/**
* A bolt that writes event message to EventHub.
*/
public class EventHubBolt extends BaseRichBolt {
- private static final long serialVersionUID = 1L;
- private static final Logger logger = LoggerFactory
- .getLogger(EventHubBolt.class);
-
- protected OutputCollector collector;
- protected PartitionSender sender;
- protected EventHubClient ehClient;
- protected EventHubBoltConfig boltConfig;
-
- public EventHubBolt(String connectionString, String entityPath) {
- boltConfig = new EventHubBoltConfig(connectionString, entityPath);
- }
-
- public EventHubBolt(String userName, String password, String namespace,
- String entityPath, boolean partitionMode) {
- boltConfig = new EventHubBoltConfig(userName, password, namespace,
- entityPath, partitionMode);
- }
-
- public EventHubBolt(EventHubBoltConfig config) {
- boltConfig = config;
- }
-
- @Override
- public void prepare(Map<String, Object> config, TopologyContext context,
- OutputCollector collector) {
- this.collector = collector;
- String myPartitionId = null;
- if (boltConfig.getPartitionMode()) {
- // We can use the task index (starting from 0) as the partition ID
- myPartitionId = "" + context.getThisTaskIndex();
- }
- logger.info("creating sender: " + boltConfig.getConnectionString()
- + ", " + boltConfig.getEntityPath() + ", " + myPartitionId);
- try {
- ehClient = EventHubClient.createFromConnectionStringSync(boltConfig.getConnectionString());
- if (boltConfig.getPartitionMode()) {
- sender = ehClient.createPartitionSenderSync(Integer.toString(context.getThisTaskIndex()));
- }
- } catch (Exception ex) {
- collector.reportError(ex);
- throw new RuntimeException(ex);
- }
-
- }
-
- @Override
- public void execute(Tuple tuple) {
- try {
- EventData sendEvent = new EventData(boltConfig.getEventDataFormat().serialize(tuple));
- if (boltConfig.getPartitionMode() && sender!=null) {
- sender.sendSync(sendEvent);
- }
- else if (boltConfig.getPartitionMode() && sender==null) {
- throw new EventHubException("Sender is null");
- }
- else if (!boltConfig.getPartitionMode() && ehClient!=null) {
- ehClient.sendSync(sendEvent);
- }
- else if (!boltConfig.getPartitionMode() && ehClient==null) {
- throw new EventHubException("ehclient is null");
- }
- collector.ack(tuple);
- } catch (EventHubException ex ) {
- collector.reportError(ex);
- collector.fail(tuple);
- } catch (ServiceBusException e) {
- collector.reportError(e);
- collector.fail(tuple);
- }
- }
-
- @Override
- public void cleanup() {
- if(sender != null) {
- try {
- sender.close().whenComplete((voidargs,error)->{
- try{
- if(error!=null){
- logger.error("Exception during sender cleanup phase"+error.toString());
- }
- ehClient.closeSync();
- }catch (Exception e){
- logger.error("Exception during ehclient cleanup phase"+e.toString());
- }
- }).get();
- } catch (InterruptedException e) {
- logger.error("Exception occured during cleanup phase"+e.toString());
- } catch (ExecutionException e) {
- logger.error("Exception occured during cleanup phase"+e.toString());
- }
- logger.info("Eventhub Bolt cleaned up");
- sender = null;
- ehClient = null;
- }
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
-
- }
+ private static final long serialVersionUID = 1L;
+ private static final Logger logger = LoggerFactory
+ .getLogger(EventHubBolt.class);
+
+ protected OutputCollector collector;
+ protected PartitionSender sender;
+ protected EventHubClient ehClient;
+ protected EventHubBoltConfig boltConfig;
+
+ public EventHubBolt(String connectionString, String entityPath) {
+ boltConfig = new EventHubBoltConfig(connectionString, entityPath);
+ }
+
+ public EventHubBolt(String userName, String password, String namespace,
+ String entityPath, boolean partitionMode) {
+ boltConfig = new EventHubBoltConfig(userName, password, namespace,
+ entityPath, partitionMode);
+ }
+
+ public EventHubBolt(EventHubBoltConfig config) {
+ boltConfig = config;
+ }
+
+ @Override
+ public void prepare(Map<String, Object> config, TopologyContext context,
+ OutputCollector collector) {
+ this.collector = collector;
+ String myPartitionId = null;
+ if (boltConfig.getPartitionMode()) {
+ // We can use the task index (starting from 0) as the partition ID
+ myPartitionId = "" + context.getThisTaskIndex();
+ }
+ logger.info("creating sender: " + boltConfig.getConnectionString()
+ + ", " + boltConfig.getEntityPath() + ", " + myPartitionId);
+ try {
+ ehClient = EventHubClient.createFromConnectionStringSync(boltConfig.getConnectionString());
+ if (boltConfig.getPartitionMode()) {
+ sender = ehClient.createPartitionSenderSync(Integer.toString(context.getThisTaskIndex()));
+ }
+ } catch (Exception ex) {
+ collector.reportError(ex);
+ throw new RuntimeException(ex);
+ }
+
+ }
+
+ @Override
+ public void execute(Tuple tuple) {
+ try {
+ EventData sendEvent = new EventData(boltConfig.getEventDataFormat().serialize(tuple));
+ if (boltConfig.getPartitionMode() && sender != null) {
+ sender.sendSync(sendEvent);
+ } else if (boltConfig.getPartitionMode() && sender == null) {
+ throw new EventHubException("Sender is null");
+ } else if (!boltConfig.getPartitionMode() && ehClient != null) {
+ ehClient.sendSync(sendEvent);
+ } else if (!boltConfig.getPartitionMode() && ehClient == null) {
+ throw new EventHubException("ehclient is null");
+ }
+ collector.ack(tuple);
+ } catch (EventHubException ex) {
+ collector.reportError(ex);
+ collector.fail(tuple);
+ } catch (ServiceBusException e) {
+ collector.reportError(e);
+ collector.fail(tuple);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ if (sender != null) {
+ try {
+ sender.close().whenComplete((voidargs, error) -> {
+ try {
+ if (error != null) {
+ logger.error("Exception during sender cleanup phase" + error.toString());
+ }
+ ehClient.closeSync();
+ } catch (Exception e) {
+ logger.error("Exception during ehclient cleanup phase" + e.toString());
+ }
+ }).get();
+ } catch (InterruptedException e) {
+ logger.error("Exception occured during cleanup phase" + e.toString());
+ } catch (ExecutionException e) {
+ logger.error("Exception occured during cleanup phase" + e.toString());
+ }
+ logger.info("Eventhub Bolt cleaned up");
+ sender = null;
+ ehClient = null;
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBoltConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBoltConfig.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBoltConfig.java
index f5e1458..41e39e4 100644
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBoltConfig.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/EventHubBoltConfig.java
@@ -15,97 +15,94 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.bolt;
import com.microsoft.azure.servicebus.ConnectionStringBuilder;
-import org.apache.storm.eventhubs.spout.EventHubSpoutConfig;
-
-import java.io.Serializable;
-
import java.io.Serializable;
+import org.apache.storm.eventhubs.spout.EventHubSpoutConfig;
/*
* EventHubs bolt configurations
*
* Partition mode:
- * With partitionMode=true you need to create the same number of tasks as the number of
+ * With partitionMode=true you need to create the same number of tasks as the number of
* EventHubs partitions, and each bolt task will only send data to one partition.
* The partition ID is the task ID of the bolt.
- *
+ *
* Event format:
* The formatter to convert tuple to bytes for EventHubs.
* if null, the default format is common delimited tuple fields.
*/
public class EventHubBoltConfig implements Serializable {
- private static final long serialVersionUID = 1L;
-
- private String connectionString;
- private final String entityPath;
- protected boolean partitionMode;
- protected IEventDataFormat dataFormat;
-
- public EventHubBoltConfig(String connectionString, String entityPath) {
- this(connectionString, entityPath, false, null);
- }
-
- public EventHubBoltConfig(String connectionString, String entityPath,
- boolean partitionMode) {
- this(connectionString, entityPath, partitionMode, null);
- }
-
- public EventHubBoltConfig(String userName, String password, String namespace,
- String entityPath, boolean partitionMode) {
- this(userName, password, namespace,
- EventHubSpoutConfig.EH_SERVICE_FQDN_SUFFIX, entityPath, partitionMode);
- }
-
- public EventHubBoltConfig(String connectionString, String entityPath,
- boolean partitionMode, IEventDataFormat dataFormat) {
- this.connectionString = connectionString;
- this.entityPath = entityPath;
- this.partitionMode = partitionMode;
- this.dataFormat = dataFormat;
- if(this.dataFormat == null) {
- this.dataFormat = new DefaultEventDataFormat();
+ private static final long serialVersionUID = 1L;
+ private final String entityPath;
+ protected boolean partitionMode;
+ protected IEventDataFormat dataFormat;
+ private String connectionString;
+
+ public EventHubBoltConfig(String connectionString, String entityPath) {
+ this(connectionString, entityPath, false, null);
+ }
+
+ public EventHubBoltConfig(String connectionString, String entityPath,
+ boolean partitionMode) {
+ this(connectionString, entityPath, partitionMode, null);
+ }
+
+ public EventHubBoltConfig(String userName, String password, String namespace,
+ String entityPath, boolean partitionMode) {
+ this(userName, password, namespace,
+ EventHubSpoutConfig.EH_SERVICE_FQDN_SUFFIX, entityPath, partitionMode);
+ }
+
+ public EventHubBoltConfig(String connectionString, String entityPath,
+ boolean partitionMode, IEventDataFormat dataFormat) {
+ this.connectionString = connectionString;
+ this.entityPath = entityPath;
+ this.partitionMode = partitionMode;
+ this.dataFormat = dataFormat;
+ if (this.dataFormat == null) {
+ this.dataFormat = new DefaultEventDataFormat();
+ }
}
- }
-
- public EventHubBoltConfig(String userName, String password, String namespace,
- String targetFqnAddress, String entityPath) {
- this(userName, password, namespace, targetFqnAddress, entityPath, false, null);
- }
-
- public EventHubBoltConfig(String userName, String password, String namespace,
- String targetFqnAddress, String entityPath, boolean partitionMode) {
- this(userName, password, namespace, targetFqnAddress, entityPath, partitionMode, null);
- }
-
- public EventHubBoltConfig(String userName, String password, String namespace,
- String targetFqnAddress, String entityPath, boolean partitionMode,
- IEventDataFormat dataFormat) {
- this.connectionString = new ConnectionStringBuilder(namespace,entityPath,
- userName,password).toString();
- this.entityPath = entityPath;
- this.partitionMode = partitionMode;
- this.dataFormat = dataFormat;
- if(this.dataFormat == null) {
- this.dataFormat = new DefaultEventDataFormat();
+
+ public EventHubBoltConfig(String userName, String password, String namespace,
+ String targetFqnAddress, String entityPath) {
+ this(userName, password, namespace, targetFqnAddress, entityPath, false, null);
+ }
+
+ public EventHubBoltConfig(String userName, String password, String namespace,
+ String targetFqnAddress, String entityPath, boolean partitionMode) {
+ this(userName, password, namespace, targetFqnAddress, entityPath, partitionMode, null);
+ }
+
+ public EventHubBoltConfig(String userName, String password, String namespace,
+ String targetFqnAddress, String entityPath, boolean partitionMode,
+ IEventDataFormat dataFormat) {
+ this.connectionString = new ConnectionStringBuilder(namespace, entityPath,
+ userName, password).toString();
+ this.entityPath = entityPath;
+ this.partitionMode = partitionMode;
+ this.dataFormat = dataFormat;
+ if (this.dataFormat == null) {
+ this.dataFormat = new DefaultEventDataFormat();
+ }
+ }
+
+ public String getConnectionString() {
+ return connectionString;
+ }
+
+ public String getEntityPath() {
+ return entityPath;
+ }
+
+ public boolean getPartitionMode() {
+ return partitionMode;
+ }
+
+ public IEventDataFormat getEventDataFormat() {
+ return dataFormat;
}
- }
-
- public String getConnectionString() {
- return connectionString;
- }
-
- public String getEntityPath() {
- return entityPath;
- }
-
- public boolean getPartitionMode() {
- return partitionMode;
- }
-
- public IEventDataFormat getEventDataFormat() {
- return dataFormat;
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/IEventDataFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/IEventDataFormat.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/IEventDataFormat.java
index d2aacb7..ec09460 100644
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/IEventDataFormat.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/bolt/IEventDataFormat.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.bolt;
import java.io.Serializable;
@@ -24,5 +25,5 @@ import org.apache.storm.tuple.Tuple;
* Serialize a tuple to a byte array to be sent to EventHubs
*/
public interface IEventDataFormat extends Serializable {
- public byte[] serialize(Tuple tuple);
+ public byte[] serialize(Tuple tuple);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/BinaryEventDataScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/BinaryEventDataScheme.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/BinaryEventDataScheme.java
index bbd46ea..4c8e0a2 100644
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/BinaryEventDataScheme.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/BinaryEventDataScheme.java
@@ -15,60 +15,58 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import com.microsoft.azure.eventhubs.EventData;
-import org.apache.storm.tuple.Fields;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.storm.tuple.Fields;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* An Event Data Scheme which deserializes message payload into the raw bytes.
*
- * The resulting tuple would contain three items, the first being the message
- * bytes, and the second a map of properties that include metadata, which can be
- * used to determine who processes the message, and how it is processed.The third is
- * the system properties which exposes information like enqueue-time, offset and
- * sequence number
+ * The resulting tuple would contain three items, the first being the message bytes, and the second a map of properties that include
+ * metadata, which can be used to determine who processes the message, and how it is processed.The third is the system properties which
+ * exposes information like enqueue-time, offset and sequence number
*/
public class BinaryEventDataScheme implements IEventDataScheme {
- private static final Logger logger = LoggerFactory.getLogger(BinaryEventDataScheme.class);
- @Override
- public List<Object> deserialize(EventData eventData){
- final List<Object> fieldContents = new ArrayList<Object>();
- byte [] messageData = null;
- if (eventData.getBytes() != null) {
- messageData = eventData.getBytes();
- }
- else if (eventData.getObject()!=null) {
- try {
- messageData = SerializeDeserializeUtil.serialize(eventData.getObject());
- } catch (IOException e) {
- logger.error("Failed to serialize EventData payload class"
- + eventData.getObject().getClass());
- logger.error("Exception encountered while serializing EventData payload is"
- + e.toString());
- throw new RuntimeException(e);
- }
- }
- Map<String, Object> metaDataMap = eventData.getProperties();
- Map<String, Object> systemMetaDataMap = eventData.getSystemProperties();
- fieldContents.add(messageData);
- fieldContents.add(metaDataMap);
- fieldContents.add(systemMetaDataMap);
- return fieldContents;
- }
+ private static final Logger logger = LoggerFactory.getLogger(BinaryEventDataScheme.class);
+
+ @Override
+ public List<Object> deserialize(EventData eventData) {
+ final List<Object> fieldContents = new ArrayList<Object>();
+ byte[] messageData = null;
+ if (eventData.getBytes() != null) {
+ messageData = eventData.getBytes();
+ } else if (eventData.getObject() != null) {
+ try {
+ messageData = SerializeDeserializeUtil.serialize(eventData.getObject());
+ } catch (IOException e) {
+ logger.error("Failed to serialize EventData payload class"
+ + eventData.getObject().getClass());
+ logger.error("Exception encountered while serializing EventData payload is"
+ + e.toString());
+ throw new RuntimeException(e);
+ }
+ }
+ Map<String, Object> metaDataMap = eventData.getProperties();
+ Map<String, Object> systemMetaDataMap = eventData.getSystemProperties();
+ fieldContents.add(messageData);
+ fieldContents.add(metaDataMap);
+ fieldContents.add(systemMetaDataMap);
+ return fieldContents;
+ }
- @Override
- public Fields getOutputFields() {
- return new Fields(FieldConstants.Message, FieldConstants.META_DATA,
- FieldConstants.SYSTEM_META_DATA);
- }
+ @Override
+ public Fields getOutputFields() {
+ return new Fields(FieldConstants.Message, FieldConstants.META_DATA,
+ FieldConstants.SYSTEM_META_DATA);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataScheme.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataScheme.java
index 9fbcecf..9bd0c22 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataScheme.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataScheme.java
@@ -15,64 +15,62 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import com.microsoft.azure.eventhubs.EventData;
-import org.apache.storm.tuple.Fields;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.storm.tuple.Fields;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * An Event Data Scheme which deserializes message payload into the Strings. No
- * encoding is assumed. The receiver will need to handle parsing of the string
- * data in appropriate encoding.
+ * An Event Data Scheme which deserializes message payload into the Strings. No encoding is assumed. The receiver will need to handle
+ * parsing of the string data in appropriate encoding.
*
- * The resulting tuple would contain two items: the the message string, and a
- * map of properties that include metadata, which can be used to determine who
- * processes the message, and how it is processed.
- *
- * For passing the raw bytes of a messsage to Bolts, refer to
- * {@link BinaryEventDataScheme}.
+ * The resulting tuple would contain two items: the the message string, and a map of properties that include metadata, which can be used to
+ * determine who processes the message, and how it is processed.
+ *
+ * For passing the raw bytes of a messsage to Bolts, refer to {@link BinaryEventDataScheme}.
*/
public class EventDataScheme implements IEventDataScheme {
- private static final long serialVersionUID = 1L;
- private static final Logger logger = LoggerFactory.getLogger(EventDataScheme.class);
- @Override
- public List<Object> deserialize(EventData eventData) {
- final List<Object> fieldContents = new ArrayList<Object>();
- String messageData = "";
- if (eventData.getBytes()!=null) {
- messageData = new String(eventData.getBytes());
- }
- /*Will only serialize AMQPValue type*/
- else if (eventData.getObject()!=null) {
- try {
- if (!(eventData.getObject() instanceof List)) {
- messageData = eventData.getObject().toString();
- } else {
- throw new RuntimeException("Cannot serialize the given AMQP type");
- }
- } catch (RuntimeException e) {
- logger.error("Failed to serialize EventData payload class"
- + eventData.getObject().getClass());
- logger.error("Exception encountered while serializing EventData payload is"
- + e.toString());
- throw e;
- }
- }
- Map<String, Object> metaDataMap = eventData.getProperties();
- fieldContents.add(messageData);
- fieldContents.add(metaDataMap);
- return fieldContents;
- }
+ private static final long serialVersionUID = 1L;
+ private static final Logger logger = LoggerFactory.getLogger(EventDataScheme.class);
+
+ @Override
+ public List<Object> deserialize(EventData eventData) {
+ final List<Object> fieldContents = new ArrayList<Object>();
+ String messageData = "";
+ if (eventData.getBytes() != null) {
+ messageData = new String(eventData.getBytes());
+ }
+ /*Will only serialize AMQPValue type*/
+ else if (eventData.getObject() != null) {
+ try {
+ if (!(eventData.getObject() instanceof List)) {
+ messageData = eventData.getObject().toString();
+ } else {
+ throw new RuntimeException("Cannot serialize the given AMQP type");
+ }
+ } catch (RuntimeException e) {
+ logger.error("Failed to serialize EventData payload class"
+ + eventData.getObject().getClass());
+ logger.error("Exception encountered while serializing EventData payload is"
+ + e.toString());
+ throw e;
+ }
+ }
+ Map<String, Object> metaDataMap = eventData.getProperties();
+ fieldContents.add(messageData);
+ fieldContents.add(metaDataMap);
+ return fieldContents;
+ }
- @Override
- public Fields getOutputFields() {
- return new Fields(FieldConstants.Message, FieldConstants.META_DATA);
- }
+ @Override
+ public Fields getOutputFields() {
+ return new Fields(FieldConstants.Message, FieldConstants.META_DATA);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataWrap.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataWrap.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataWrap.java
index 5eeb4d2..fc23c05 100644
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataWrap.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventDataWrap.java
@@ -15,34 +15,35 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import com.microsoft.azure.eventhubs.EventData;
public class EventDataWrap implements Comparable<EventDataWrap> {
- private final EventData eventData;
- private final MessageId messageId;
-
- public EventDataWrap(EventData eventdata, MessageId messageId) {
- this.eventData = eventdata;
- this.messageId = messageId;
- }
-
- public static EventDataWrap create(EventData eventData, MessageId messageId) {
- return new EventDataWrap(eventData, messageId);
- }
-
- public EventData getEventData() {
- return this.eventData;
- }
-
- public MessageId getMessageId() {
- return this.messageId;
- }
-
- @Override
- public int compareTo(EventDataWrap ed) {
- return messageId.getSequenceNumber().
- compareTo(ed.getMessageId().getSequenceNumber());
- }
+ private final EventData eventData;
+ private final MessageId messageId;
+
+ public EventDataWrap(EventData eventdata, MessageId messageId) {
+ this.eventData = eventdata;
+ this.messageId = messageId;
+ }
+
+ public static EventDataWrap create(EventData eventData, MessageId messageId) {
+ return new EventDataWrap(eventData, messageId);
+ }
+
+ public EventData getEventData() {
+ return this.eventData;
+ }
+
+ public MessageId getMessageId() {
+ return this.messageId;
+ }
+
+ @Override
+ public int compareTo(EventDataWrap ed) {
+ return messageId.getSequenceNumber().
+ compareTo(ed.getMessageId().getSequenceNumber());
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubFilter.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubFilter.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubFilter.java
index a375380..509191d 100644
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubFilter.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubFilter.java
@@ -15,32 +15,33 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import java.time.Instant;
-public class EventHubFilter implements IEventFilter{
+public class EventHubFilter implements IEventFilter {
String offset = null;
Instant time = null;
- public EventHubFilter(String offset){
+ public EventHubFilter(String offset) {
this.offset = offset;
this.time = null;
}
- public EventHubFilter(Instant time){
+ public EventHubFilter(Instant time) {
this.time = time;
this.offset = null;
}
@Override
- public String getOffset(){
+ public String getOffset() {
return offset;
}
@Override
- public Instant getTime(){
+ public Instant getTime() {
return time;
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubReceiverImpl.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubReceiverImpl.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubReceiverImpl.java
index 459b9bc..83dc850 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubReceiverImpl.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubReceiverImpl.java
@@ -15,152 +15,149 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import com.microsoft.azure.eventhubs.EventData;
import com.microsoft.azure.eventhubs.EventHubClient;
import com.microsoft.azure.eventhubs.PartitionReceiver;
import com.microsoft.azure.servicebus.ServiceBusException;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
import org.apache.storm.metric.api.CountMetric;
import org.apache.storm.metric.api.MeanReducer;
import org.apache.storm.metric.api.ReducedMetric;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-
public class EventHubReceiverImpl implements IEventHubReceiver {
- private static final Logger logger = LoggerFactory.getLogger(EventHubReceiverImpl.class);
+ private static final Logger logger = LoggerFactory.getLogger(EventHubReceiverImpl.class);
- private final String connectionString;
- private final String entityName;
- private final String partitionId;
- private final String consumerGroupName;
+ private final String connectionString;
+ private final String entityName;
+ private final String partitionId;
+ private final String consumerGroupName;
- private PartitionReceiver receiver;
- private EventHubClient ehClient;
- private ReducedMetric receiveApiLatencyMean;
- private CountMetric receiveApiCallCount;
- private CountMetric receiveMessageCount;
+ private PartitionReceiver receiver;
+ private EventHubClient ehClient;
+ private ReducedMetric receiveApiLatencyMean;
+ private CountMetric receiveApiCallCount;
+ private CountMetric receiveMessageCount;
- public EventHubReceiverImpl(EventHubSpoutConfig config, String partitionId) {
- this.connectionString = config.getConnectionString();
- this.entityName = config.getEntityPath();
- this.partitionId = partitionId;
- this.consumerGroupName = config.getConsumerGroupName();
- receiveApiLatencyMean = new ReducedMetric(new MeanReducer());
- receiveApiCallCount = new CountMetric();
- receiveMessageCount = new CountMetric();
- }
+ public EventHubReceiverImpl(EventHubSpoutConfig config, String partitionId) {
+ this.connectionString = config.getConnectionString();
+ this.entityName = config.getEntityPath();
+ this.partitionId = partitionId;
+ this.consumerGroupName = config.getConsumerGroupName();
+ receiveApiLatencyMean = new ReducedMetric(new MeanReducer());
+ receiveApiCallCount = new CountMetric();
+ receiveMessageCount = new CountMetric();
+ }
- @Override
- public void open(IEventFilter filter) throws EventHubException {
- logger.info("creating eventhub receiver: partitionId=" + partitionId +
- ", filter=" + filter.getOffset() != null ?
- filter.getOffset() : Long.toString(filter.getTime().toEpochMilli()));
- long start = System.currentTimeMillis();
- try {
- ehClient = EventHubClient.createFromConnectionStringSync(connectionString);
+ @Override
+ public void open(IEventFilter filter) throws EventHubException {
+ logger.info("creating eventhub receiver: partitionId=" + partitionId +
+ ", filter=" + filter.getOffset() != null ?
+ filter.getOffset() : Long.toString(filter.getTime().toEpochMilli()));
+ long start = System.currentTimeMillis();
+ try {
+ ehClient = EventHubClient.createFromConnectionStringSync(connectionString);
- if (filter.getOffset()!=null) {
- receiver = ehClient.createEpochReceiverSync(
- consumerGroupName,
- partitionId,
- filter.getOffset(),
- false,
- 1);
- }
- else if (filter.getTime()!=null) {
- receiver = ehClient.createEpochReceiverSync(
- consumerGroupName,
- partitionId,
- filter.getTime(),
- 1);
- }
- else{
- throw new RuntimeException("Eventhub receiver must have " +
- "an offset or time to be created");
- }
- } catch (IOException e) {
- logger.error("Exception in creating ehclient"+ e.toString());
- throw new EventHubException(e);
- }
- catch (ServiceBusException e) {
- logger.error("Exception in creating Receiver"+e.toString());
- throw new EventHubException(e);
+ if (filter.getOffset() != null) {
+ receiver = ehClient.createEpochReceiverSync(
+ consumerGroupName,
+ partitionId,
+ filter.getOffset(),
+ false,
+ 1);
+ } else if (filter.getTime() != null) {
+ receiver = ehClient.createEpochReceiverSync(
+ consumerGroupName,
+ partitionId,
+ filter.getTime(),
+ 1);
+ } else {
+ throw new RuntimeException("Eventhub receiver must have " +
+ "an offset or time to be created");
+ }
+ } catch (IOException e) {
+ logger.error("Exception in creating ehclient" + e.toString());
+ throw new EventHubException(e);
+ } catch (ServiceBusException e) {
+ logger.error("Exception in creating Receiver" + e.toString());
+ throw new EventHubException(e);
+ }
+ long end = System.currentTimeMillis();
+ logger.info("created eventhub receiver, time taken(ms): " + (end - start));
}
- long end = System.currentTimeMillis();
- logger.info("created eventhub receiver, time taken(ms): " + (end-start));
- }
- @Override
- public void close() {
- if(receiver != null) {
- try {
- receiver.close().whenComplete((voidargs,error)->{
- try {
- if (error!=null) {
- logger.error("Exception during receiver close phase"+error.toString());
+ @Override
+ public void close() {
+ if (receiver != null) {
+ try {
+ receiver.close().whenComplete((voidargs, error) -> {
+ try {
+ if (error != null) {
+ logger.error("Exception during receiver close phase" + error.toString());
+ }
+ ehClient.closeSync();
+ } catch (Exception e) {
+ logger.error("Exception during ehclient close phase" + e.toString());
+ }
+ }).get();
+ } catch (InterruptedException e) {
+ logger.error("Exception occured during close phase" + e.toString());
+ } catch (ExecutionException e) {
+ logger.error("Exception occured during close phase" + e.toString());
}
- ehClient.closeSync();
- } catch (Exception e) {
- logger.error("Exception during ehclient close phase"+e.toString());
- }
- }).get();
- } catch (InterruptedException e) {
- logger.error("Exception occured during close phase"+e.toString());
- } catch (ExecutionException e) {
- logger.error("Exception occured during close phase"+e.toString());
- }
- logger.info("closed eventhub receiver: partitionId=" + partitionId );
- receiver = null;
- ehClient = null;
+ logger.info("closed eventhub receiver: partitionId=" + partitionId);
+ receiver = null;
+ ehClient = null;
+ }
}
- }
-
- @Override
- public boolean isOpen() {
- return (receiver != null);
- }
- @Override
- public EventDataWrap receive() {
- long start = System.currentTimeMillis();
- Iterable<EventData> receivedEvents=null;
- /*Get one message at a time for backward compatibility behaviour*/
- try {
- receivedEvents = receiver.receiveSync(1);
- } catch (ServiceBusException e) {
- logger.error("Exception occured during receive"+e.toString());
- return null;
+ @Override
+ public boolean isOpen() {
+ return (receiver != null);
}
- long end = System.currentTimeMillis();
- long millis = (end - start);
- receiveApiLatencyMean.update(millis);
- receiveApiCallCount.incr();
- if (receivedEvents == null || receivedEvents.spliterator().getExactSizeIfKnown() == 0) {
- return null;
- }
- receiveMessageCount.incr();
- EventData receivedEvent = receivedEvents.iterator().next();
- MessageId messageId = new MessageId(partitionId,
- receivedEvent.getSystemProperties().getOffset(),
- receivedEvent.getSystemProperties().getSequenceNumber());
+ @Override
+ public EventDataWrap receive() {
+ long start = System.currentTimeMillis();
+ Iterable<EventData> receivedEvents = null;
+ /*Get one message at a time for backward compatibility behaviour*/
+ try {
+ receivedEvents = receiver.receiveSync(1);
+ } catch (ServiceBusException e) {
+ logger.error("Exception occured during receive" + e.toString());
+ return null;
+ }
+ long end = System.currentTimeMillis();
+ long millis = (end - start);
+ receiveApiLatencyMean.update(millis);
+ receiveApiCallCount.incr();
+
+ if (receivedEvents == null || receivedEvents.spliterator().getExactSizeIfKnown() == 0) {
+ return null;
+ }
+ receiveMessageCount.incr();
+ EventData receivedEvent = receivedEvents.iterator().next();
+ MessageId messageId = new MessageId(partitionId,
+ receivedEvent.getSystemProperties().getOffset(),
+ receivedEvent.getSystemProperties().getSequenceNumber());
- return EventDataWrap.create(receivedEvent,messageId);
- }
+ return EventDataWrap.create(receivedEvent, messageId);
+ }
- @Override
- public Map<String, Object> getMetricsData() {
- Map<String, Object> ret = new HashMap<>();
- ret.put(partitionId + "/receiveApiLatencyMean", receiveApiLatencyMean.getValueAndReset());
- ret.put(partitionId + "/receiveApiCallCount", receiveApiCallCount.getValueAndReset());
- ret.put(partitionId + "/receiveMessageCount", receiveMessageCount.getValueAndReset());
- return ret;
- }
+ @Override
+ public Map<String, Object> getMetricsData() {
+ Map<String, Object> ret = new HashMap<>();
+ ret.put(partitionId + "/receiveApiLatencyMean", receiveApiLatencyMean.getValueAndReset());
+ ret.put(partitionId + "/receiveApiCallCount", receiveApiCallCount.getValueAndReset());
+ ret.put(partitionId + "/receiveMessageCount", receiveMessageCount.getValueAndReset());
+ return ret;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubSpout.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubSpout.java
index d8c3d09..f8e144e 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubSpout.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubSpout.java
@@ -15,9 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import com.google.common.base.Strings;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
import org.apache.storm.Config;
import org.apache.storm.metric.api.IMetric;
import org.apache.storm.spout.SpoutOutputCollector;
@@ -27,236 +32,233 @@ import org.apache.storm.topology.base.BaseRichSpout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
public class EventHubSpout extends BaseRichSpout {
- private static final Logger logger = LoggerFactory.getLogger(EventHubSpout.class);
-
- private final UUID instanceId;
- private final EventHubSpoutConfig eventHubConfig;
- private final IEventDataScheme scheme;
- private final int checkpointIntervalInSeconds;
-
- private IStateStore stateStore;
- private IPartitionCoordinator partitionCoordinator;
- private IPartitionManagerFactory pmFactory;
- private IEventHubReceiverFactory recvFactory;
- private SpoutOutputCollector collector;
- private long lastCheckpointTime;
- private int currentPartitionIndex = -1;
-
- public EventHubSpout(String username, String password, String namespace,
- String entityPath, int partitionCount) {
- this(new EventHubSpoutConfig(username, password, namespace, entityPath, partitionCount));
- }
-
- public EventHubSpout(EventHubSpoutConfig spoutConfig) {
- this(spoutConfig, null, null, null);
- }
-
- public EventHubSpout(EventHubSpoutConfig spoutConfig,
- IStateStore store,
- IPartitionManagerFactory pmFactory,
- IEventHubReceiverFactory recvFactory) {
- this.eventHubConfig = spoutConfig;
- this.scheme = spoutConfig.getEventDataScheme();
- this.instanceId = UUID.randomUUID();
- this.checkpointIntervalInSeconds = spoutConfig.getCheckpointIntervalInSeconds();
- this.lastCheckpointTime = System.currentTimeMillis();
- stateStore = store;
- this.pmFactory = pmFactory;
- if(this.pmFactory == null) {
- this.pmFactory = new IPartitionManagerFactory() {
- @Override
- public IPartitionManager create(EventHubSpoutConfig spoutConfig,
- String partitionId, IStateStore stateStore,
- IEventHubReceiver receiver) {
- return new PartitionManager(spoutConfig, partitionId,
- stateStore, receiver);
+ private static final Logger logger = LoggerFactory.getLogger(EventHubSpout.class);
+
+ private final UUID instanceId;
+ private final EventHubSpoutConfig eventHubConfig;
+ private final IEventDataScheme scheme;
+ private final int checkpointIntervalInSeconds;
+
+ private IStateStore stateStore;
+ private IPartitionCoordinator partitionCoordinator;
+ private IPartitionManagerFactory pmFactory;
+ private IEventHubReceiverFactory recvFactory;
+ private SpoutOutputCollector collector;
+ private long lastCheckpointTime;
+ private int currentPartitionIndex = -1;
+
+ public EventHubSpout(String username, String password, String namespace,
+ String entityPath, int partitionCount) {
+ this(new EventHubSpoutConfig(username, password, namespace, entityPath, partitionCount));
+ }
+
+ public EventHubSpout(EventHubSpoutConfig spoutConfig) {
+ this(spoutConfig, null, null, null);
+ }
+
+ public EventHubSpout(EventHubSpoutConfig spoutConfig,
+ IStateStore store,
+ IPartitionManagerFactory pmFactory,
+ IEventHubReceiverFactory recvFactory) {
+ this.eventHubConfig = spoutConfig;
+ this.scheme = spoutConfig.getEventDataScheme();
+ this.instanceId = UUID.randomUUID();
+ this.checkpointIntervalInSeconds = spoutConfig.getCheckpointIntervalInSeconds();
+ this.lastCheckpointTime = System.currentTimeMillis();
+ stateStore = store;
+ this.pmFactory = pmFactory;
+ if (this.pmFactory == null) {
+ this.pmFactory = new IPartitionManagerFactory() {
+ @Override
+ public IPartitionManager create(EventHubSpoutConfig spoutConfig,
+ String partitionId, IStateStore stateStore,
+ IEventHubReceiver receiver) {
+ return new PartitionManager(spoutConfig, partitionId,
+ stateStore, receiver);
+ }
+ };
+ }
+ this.recvFactory = recvFactory;
+ if (this.recvFactory == null) {
+ this.recvFactory = new IEventHubReceiverFactory() {
+ @Override
+ public IEventHubReceiver create(EventHubSpoutConfig spoutConfig,
+ String partitionId) {
+ return new EventHubReceiverImpl(spoutConfig, partitionId);
+ }
+ };
}
- };
+
}
- this.recvFactory = recvFactory;
- if(this.recvFactory == null) {
- this.recvFactory = new IEventHubReceiverFactory() {
- @Override
- public IEventHubReceiver create(EventHubSpoutConfig spoutConfig,
- String partitionId) {
- return new EventHubReceiverImpl(spoutConfig, partitionId);
+
+ /**
+ * This is a extracted method that is easy to test
+ *
+ * @param config
+ * @param totalTasks
+ * @param taskIndex
+ * @param collector
+ * @throws Exception
+ */
+ public void preparePartitions(Map<String, Object> config, int totalTasks, int taskIndex, SpoutOutputCollector collector) throws
+ Exception {
+ this.collector = collector;
+ if (stateStore == null) {
+ String zkEndpointAddress = eventHubConfig.getZkConnectionString();
+ if (zkEndpointAddress == null || zkEndpointAddress.length() == 0) {
+ //use storm's zookeeper servers if not specified.
+ @SuppressWarnings("unchecked")
+ List<String> zkServers = (List<String>) config.get(Config.STORM_ZOOKEEPER_SERVERS);
+ Integer zkPort = ((Number) config.get(Config.STORM_ZOOKEEPER_PORT)).intValue();
+ StringBuilder sb = new StringBuilder();
+ for (String zk : zkServers) {
+ if (sb.length() > 0) {
+ sb.append(',');
+ }
+ sb.append(zk + ":" + zkPort);
+ }
+ zkEndpointAddress = sb.toString();
+ }
+ stateStore = new ZookeeperStateStore(zkEndpointAddress,
+ Integer.parseInt(config.get(Config.STORM_ZOOKEEPER_RETRY_TIMES).toString()),
+ Integer.parseInt(config.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL).toString()));
+ }
+ stateStore.open();
+
+ partitionCoordinator = new StaticPartitionCoordinator(
+ eventHubConfig, taskIndex, totalTasks, stateStore, pmFactory, recvFactory);
+
+ for (IPartitionManager partitionManager :
+ partitionCoordinator.getMyPartitionManagers()) {
+ partitionManager.open();
}
- };
}
-
- }
-
- /**
- * This is a extracted method that is easy to test
- * @param config
- * @param totalTasks
- * @param taskIndex
- * @param collector
- * @throws Exception
- */
- public void preparePartitions(Map<String, Object> config, int totalTasks, int taskIndex, SpoutOutputCollector collector) throws Exception {
- this.collector = collector;
- if(stateStore == null) {
- String zkEndpointAddress = eventHubConfig.getZkConnectionString();
- if (zkEndpointAddress == null || zkEndpointAddress.length() == 0) {
- //use storm's zookeeper servers if not specified.
- @SuppressWarnings("unchecked")
- List<String> zkServers = (List<String>) config.get(Config.STORM_ZOOKEEPER_SERVERS);
- Integer zkPort = ((Number) config.get(Config.STORM_ZOOKEEPER_PORT)).intValue();
- StringBuilder sb = new StringBuilder();
- for (String zk : zkServers) {
- if (sb.length() > 0) {
- sb.append(',');
- }
- sb.append(zk+":"+zkPort);
+
+ @Override
+ public void open(Map<String, Object> config, TopologyContext context, SpoutOutputCollector collector) {
+ logger.info("begin:start open()");
+ String topologyName = (String) config.get(Config.TOPOLOGY_NAME);
+ eventHubConfig.setTopologyName(topologyName);
+
+ int totalTasks = context.getComponentTasks(context.getThisComponentId()).size();
+ int taskIndex = context.getThisTaskIndex();
+ if (totalTasks > eventHubConfig.getPartitionCount()) {
+ throw new RuntimeException("total tasks of EventHubSpout is greater than partition count.");
+ }
+
+ logger.info(String.format("topologyName: %s, totalTasks: %d, taskIndex: %d", topologyName, totalTasks, taskIndex));
+
+ try {
+ preparePartitions(config, totalTasks, taskIndex, collector);
+ } catch (Exception e) {
+ collector.reportError(e);
+ throw new RuntimeException(e);
}
- zkEndpointAddress = sb.toString();
- }
- stateStore = new ZookeeperStateStore(zkEndpointAddress,
- Integer.parseInt(config.get(Config.STORM_ZOOKEEPER_RETRY_TIMES).toString()),
- Integer.parseInt(config.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL).toString()));
+
+ //register metrics
+ context.registerMetric("EventHubReceiver", new IMetric() {
+ @Override
+ public Object getValueAndReset() {
+ Map<String, Object> concatMetricsDataMaps = new HashMap<>();
+ for (IPartitionManager partitionManager :
+ partitionCoordinator.getMyPartitionManagers()) {
+ concatMetricsDataMaps.putAll(partitionManager.getMetricsData());
+ }
+ return concatMetricsDataMaps;
+ }
+ }, Integer.parseInt(config.get(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS).toString()));
+ logger.info("end open()");
}
- stateStore.open();
- partitionCoordinator = new StaticPartitionCoordinator(
- eventHubConfig, taskIndex, totalTasks, stateStore, pmFactory, recvFactory);
+ @Override
+ public void nextTuple() {
+ EventDataWrap eventDatawrap = null;
+
+ List<IPartitionManager> partitionManagers = partitionCoordinator.getMyPartitionManagers();
+ for (int i = 0; i < partitionManagers.size(); i++) {
+ currentPartitionIndex = (currentPartitionIndex + 1) % partitionManagers.size();
+ IPartitionManager partitionManager = partitionManagers.get(currentPartitionIndex);
+
+ if (partitionManager == null) {
+ throw new RuntimeException("partitionManager doesn't exist.");
+ }
+
+ eventDatawrap = partitionManager.receive();
+
+ if (eventDatawrap != null) {
+ break;
+ }
+ }
+
+ if (eventDatawrap != null) {
+ MessageId messageId = eventDatawrap.getMessageId();
+ List<Object> tuples = scheme.deserialize(eventDatawrap.getEventData());
+ if (tuples != null) {
+ collector.emit(tuples, messageId);
+ }
+ }
+
+ checkpointIfNeeded();
- for (IPartitionManager partitionManager :
- partitionCoordinator.getMyPartitionManagers()) {
- partitionManager.open();
+ // We don't need to sleep here because the IPartitionManager.receive() is
+ // a blocked call so it's fine to call this function in a tight loop.
}
- }
-
- @Override
- public void open(Map<String, Object> config, TopologyContext context, SpoutOutputCollector collector) {
- logger.info("begin:start open()");
- String topologyName = (String) config.get(Config.TOPOLOGY_NAME);
- eventHubConfig.setTopologyName(topologyName);
-
- int totalTasks = context.getComponentTasks(context.getThisComponentId()).size();
- int taskIndex = context.getThisTaskIndex();
- if (totalTasks > eventHubConfig.getPartitionCount()) {
- throw new RuntimeException("total tasks of EventHubSpout is greater than partition count.");
+
+ @Override
+ public void ack(Object msgId) {
+ MessageId messageId = (MessageId) msgId;
+ IPartitionManager partitionManager = partitionCoordinator.getPartitionManager(messageId.getPartitionId());
+ String offset = messageId.getOffset();
+ partitionManager.ack(offset);
}
- logger.info(String.format("topologyName: %s, totalTasks: %d, taskIndex: %d", topologyName, totalTasks, taskIndex));
+ @Override
+ public void fail(Object msgId) {
+ MessageId messageId = (MessageId) msgId;
+ IPartitionManager partitionManager = partitionCoordinator.getPartitionManager(messageId.getPartitionId());
+ String offset = messageId.getOffset();
+ partitionManager.fail(offset);
+ }
- try {
- preparePartitions(config, totalTasks, taskIndex, collector);
- } catch (Exception e) {
- collector.reportError(e);
- throw new RuntimeException(e);
+ @Override
+ public void deactivate() {
+ // let's checkpoint so that we can get the last checkpoint when restarting.
+ checkpoint();
}
-
- //register metrics
- context.registerMetric("EventHubReceiver", new IMetric() {
- @Override
- public Object getValueAndReset() {
- Map<String, Object> concatMetricsDataMaps = new HashMap<>();
- for (IPartitionManager partitionManager :
+
+ @Override
+ public void close() {
+ for (IPartitionManager partitionManager :
partitionCoordinator.getMyPartitionManagers()) {
- concatMetricsDataMaps.putAll(partitionManager.getMetricsData());
- }
- return concatMetricsDataMaps;
- }
- }, Integer.parseInt(config.get(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS).toString()));
- logger.info("end open()");
- }
-
- @Override
- public void nextTuple() {
- EventDataWrap eventDatawrap = null;
-
- List<IPartitionManager> partitionManagers = partitionCoordinator.getMyPartitionManagers();
- for (int i = 0; i < partitionManagers.size(); i++) {
- currentPartitionIndex = (currentPartitionIndex + 1) % partitionManagers.size();
- IPartitionManager partitionManager = partitionManagers.get(currentPartitionIndex);
-
- if (partitionManager == null) {
- throw new RuntimeException("partitionManager doesn't exist.");
- }
-
- eventDatawrap = partitionManager.receive();
-
- if (eventDatawrap != null) {
- break;
- }
+ partitionManager.close();
+ }
+ stateStore.close();
}
- if (eventDatawrap != null) {
- MessageId messageId = eventDatawrap.getMessageId();
- List<Object> tuples = scheme.deserialize(eventDatawrap.getEventData());
- if (tuples != null) {
- collector.emit(tuples, messageId);
- }
- }
-
- checkpointIfNeeded();
-
- // We don't need to sleep here because the IPartitionManager.receive() is
- // a blocked call so it's fine to call this function in a tight loop.
- }
-
- @Override
- public void ack(Object msgId) {
- MessageId messageId = (MessageId) msgId;
- IPartitionManager partitionManager = partitionCoordinator.getPartitionManager(messageId.getPartitionId());
- String offset = messageId.getOffset();
- partitionManager.ack(offset);
- }
-
- @Override
- public void fail(Object msgId) {
- MessageId messageId = (MessageId) msgId;
- IPartitionManager partitionManager = partitionCoordinator.getPartitionManager(messageId.getPartitionId());
- String offset = messageId.getOffset();
- partitionManager.fail(offset);
- }
-
- @Override
- public void deactivate() {
- // let's checkpoint so that we can get the last checkpoint when restarting.
- checkpoint();
- }
-
- @Override
- public void close() {
- for (IPartitionManager partitionManager :
- partitionCoordinator.getMyPartitionManagers()) {
- partitionManager.close();
- }
- stateStore.close();
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- if (Strings.isNullOrEmpty(eventHubConfig.getOutputStreamId())) {
- declarer.declare(scheme.getOutputFields());
- } else {
- declarer.declareStream(eventHubConfig.getOutputStreamId(), scheme.getOutputFields());
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ if (Strings.isNullOrEmpty(eventHubConfig.getOutputStreamId())) {
+ declarer.declare(scheme.getOutputFields());
+ } else {
+ declarer.declareStream(eventHubConfig.getOutputStreamId(), scheme.getOutputFields());
+ }
}
- }
- private void checkpointIfNeeded() {
- long nextCheckpointTime = lastCheckpointTime + checkpointIntervalInSeconds * 1000;
- if (nextCheckpointTime < System.currentTimeMillis()) {
+ private void checkpointIfNeeded() {
+ long nextCheckpointTime = lastCheckpointTime + checkpointIntervalInSeconds * 1000;
+ if (nextCheckpointTime < System.currentTimeMillis()) {
- checkpoint();
- lastCheckpointTime = System.currentTimeMillis();
+ checkpoint();
+ lastCheckpointTime = System.currentTimeMillis();
+ }
}
- }
-
- private void checkpoint() {
- for (IPartitionManager partitionManager :
- partitionCoordinator.getMyPartitionManagers()) {
- partitionManager.checkpoint();
+
+ private void checkpoint() {
+ for (IPartitionManager partitionManager :
+ partitionCoordinator.getMyPartitionManagers()) {
+ partitionManager.checkpoint();
+ }
}
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubSpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubSpoutConfig.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubSpoutConfig.java
index 5556970..cd27b11 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubSpoutConfig.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/EventHubSpoutConfig.java
@@ -15,242 +15,241 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import com.microsoft.azure.eventhubs.EventHubClient;
import com.microsoft.azure.servicebus.ConnectionStringBuilder;
-
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
public class EventHubSpoutConfig implements Serializable {
- private static final long serialVersionUID = 1L;
-
- public static final String EH_SERVICE_FQDN_SUFFIX = "servicebus.windows.net";
- private final String userName;
- private final String password;
- private final String namespace;
- private final String entityPath;
- private final int partitionCount;
-
- private String zkConnectionString = null; // if null then use zookeeper used
- // by Storm
- private int checkpointIntervalInSeconds = 10;
- private int receiverCredits = 1024;
- private int maxPendingMsgsPerPartition = 1024;
- private long enqueueTimeFilter = 0; // timestamp in millisecond, 0 means
- // disabling filter
- private String connectionString;
- private String topologyName;
- private IEventDataScheme scheme = new StringEventDataScheme();
- private String consumerGroupName = EventHubClient.DEFAULT_CONSUMER_GROUP_NAME;
- private String outputStreamId;
-
-
- // These are mandatory parameters
- public EventHubSpoutConfig(String username, String password,
- String namespace, String entityPath, int partitionCount) {
- this.userName = username;
- this.password = password;
- this.connectionString = new ConnectionStringBuilder(namespace,entityPath,
- username,password).toString();
- this.namespace = namespace;
- this.entityPath = entityPath;
- this.partitionCount = partitionCount;
- }
-
- // Keep this constructor for backward compatibility
- public EventHubSpoutConfig(String username, String password,
- String namespace, String entityPath, int partitionCount,
- String zkConnectionString) {
- this(username, password, namespace, entityPath, partitionCount);
- setZkConnectionString(zkConnectionString);
- }
-
- // Keep this constructor for backward compatibility
- public EventHubSpoutConfig(String username, String password,
- String namespace, String entityPath, int partitionCount,
- String zkConnectionString, int checkpointIntervalInSeconds,
- int receiverCredits) {
- this(username, password, namespace, entityPath, partitionCount,
- zkConnectionString);
- setCheckpointIntervalInSeconds(checkpointIntervalInSeconds);
- setReceiverCredits(receiverCredits);
- }
-
- public EventHubSpoutConfig(String username, String password,
- String namespace, String entityPath, int partitionCount,
- String zkConnectionString, int checkpointIntervalInSeconds,
- int receiverCredits, long enqueueTimeFilter) {
- this(username, password, namespace, entityPath, partitionCount,
- zkConnectionString, checkpointIntervalInSeconds,
- receiverCredits);
- setEnqueueTimeFilter(enqueueTimeFilter);
- }
-
- // Keep this constructor for backward compatibility
- public EventHubSpoutConfig(String username, String password,
- String namespace, String entityPath, int partitionCount,
- String zkConnectionString, int checkpointIntervalInSeconds,
- int receiverCredits, int maxPendingMsgsPerPartition,
- long enqueueTimeFilter) {
-
- this(username, password, namespace, entityPath, partitionCount,
- zkConnectionString, checkpointIntervalInSeconds,
- receiverCredits);
- setMaxPendingMsgsPerPartition(maxPendingMsgsPerPartition);
- setEnqueueTimeFilter(enqueueTimeFilter);
- }
-
- public String getNamespace() {
- return namespace;
- }
-
- public String getEntityPath() {
- return entityPath;
- }
-
- public int getPartitionCount() {
- return partitionCount;
- }
-
- public String getZkConnectionString() {
- return zkConnectionString;
- }
-
- public void setZkConnectionString(String value) {
- zkConnectionString = value;
- }
-
- public EventHubSpoutConfig withZkConnectionString(String value) {
- setZkConnectionString(value);
- return this;
- }
-
- public int getCheckpointIntervalInSeconds() {
- return checkpointIntervalInSeconds;
- }
-
- public void setCheckpointIntervalInSeconds(int value) {
- checkpointIntervalInSeconds = value;
- }
-
- public EventHubSpoutConfig withCheckpointIntervalInSeconds(int value) {
- setCheckpointIntervalInSeconds(value);
- return this;
- }
-
- public int getReceiverCredits() {
- return receiverCredits;
- }
-
- public void setReceiverCredits(int value) {
- receiverCredits = value;
- }
-
- public EventHubSpoutConfig withReceiverCredits(int value) {
- setReceiverCredits(value);
- return this;
- }
-
- public int getMaxPendingMsgsPerPartition() {
- return maxPendingMsgsPerPartition;
- }
-
- public void setMaxPendingMsgsPerPartition(int value) {
- maxPendingMsgsPerPartition = value;
- }
-
- public EventHubSpoutConfig withMaxPendingMsgsPerPartition(int value) {
- setMaxPendingMsgsPerPartition(value);
- return this;
- }
-
- public long getEnqueueTimeFilter() {
- return enqueueTimeFilter;
- }
-
- public void setEnqueueTimeFilter(long value) {
- enqueueTimeFilter = value;
- }
-
- public EventHubSpoutConfig withEnqueueTimeFilter(long value) {
- setEnqueueTimeFilter(value);
- return this;
- }
-
- public String getTopologyName() {
- return topologyName;
- }
-
- public void setTopologyName(String value) {
- topologyName = value;
- }
-
- public EventHubSpoutConfig withTopologyName(String value) {
- setTopologyName(value);
- return this;
- }
-
- public IEventDataScheme getEventDataScheme() {
- return scheme;
- }
-
- public void setEventDataScheme(IEventDataScheme scheme) {
- this.scheme = scheme;
- }
-
- public EventHubSpoutConfig withEventDataScheme(IEventDataScheme value) {
- setEventDataScheme(value);
- return this;
- }
-
- public String getConsumerGroupName() {
- return consumerGroupName;
- }
-
- public void setConsumerGroupName(String value) {
- consumerGroupName = value;
- }
-
- public EventHubSpoutConfig withConsumerGroupName(String value) {
- setConsumerGroupName(value);
- return this;
- }
+ public static final String EH_SERVICE_FQDN_SUFFIX = "servicebus.windows.net";
+ private static final long serialVersionUID = 1L;
+ private final String userName;
+ private final String password;
+ private final String namespace;
+ private final String entityPath;
+ private final int partitionCount;
+
+ private String zkConnectionString = null; // if null then use zookeeper used
+ // by Storm
+ private int checkpointIntervalInSeconds = 10;
+ private int receiverCredits = 1024;
+ private int maxPendingMsgsPerPartition = 1024;
+ private long enqueueTimeFilter = 0; // timestamp in millisecond, 0 means
+ // disabling filter
+ private String connectionString;
+ private String topologyName;
+ private IEventDataScheme scheme = new StringEventDataScheme();
+ private String consumerGroupName = EventHubClient.DEFAULT_CONSUMER_GROUP_NAME;
+ private String outputStreamId;
+
+
+ // These are mandatory parameters
+ public EventHubSpoutConfig(String username, String password,
+ String namespace, String entityPath, int partitionCount) {
+ this.userName = username;
+ this.password = password;
+ this.connectionString = new ConnectionStringBuilder(namespace, entityPath,
+ username, password).toString();
+ this.namespace = namespace;
+ this.entityPath = entityPath;
+ this.partitionCount = partitionCount;
+ }
+
+ // Keep this constructor for backward compatibility
+ public EventHubSpoutConfig(String username, String password,
+ String namespace, String entityPath, int partitionCount,
+ String zkConnectionString) {
+ this(username, password, namespace, entityPath, partitionCount);
+ setZkConnectionString(zkConnectionString);
+ }
+
+ // Keep this constructor for backward compatibility
+ public EventHubSpoutConfig(String username, String password,
+ String namespace, String entityPath, int partitionCount,
+ String zkConnectionString, int checkpointIntervalInSeconds,
+ int receiverCredits) {
+ this(username, password, namespace, entityPath, partitionCount,
+ zkConnectionString);
+ setCheckpointIntervalInSeconds(checkpointIntervalInSeconds);
+ setReceiverCredits(receiverCredits);
+ }
+
+ public EventHubSpoutConfig(String username, String password,
+ String namespace, String entityPath, int partitionCount,
+ String zkConnectionString, int checkpointIntervalInSeconds,
+ int receiverCredits, long enqueueTimeFilter) {
+ this(username, password, namespace, entityPath, partitionCount,
+ zkConnectionString, checkpointIntervalInSeconds,
+ receiverCredits);
+ setEnqueueTimeFilter(enqueueTimeFilter);
+ }
+
+ // Keep this constructor for backward compatibility
+ public EventHubSpoutConfig(String username, String password,
+ String namespace, String entityPath, int partitionCount,
+ String zkConnectionString, int checkpointIntervalInSeconds,
+ int receiverCredits, int maxPendingMsgsPerPartition,
+ long enqueueTimeFilter) {
+
+ this(username, password, namespace, entityPath, partitionCount,
+ zkConnectionString, checkpointIntervalInSeconds,
+ receiverCredits);
+ setMaxPendingMsgsPerPartition(maxPendingMsgsPerPartition);
+ setEnqueueTimeFilter(enqueueTimeFilter);
+ }
+
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public String getEntityPath() {
+ return entityPath;
+ }
+
+ public int getPartitionCount() {
+ return partitionCount;
+ }
+
+ public String getZkConnectionString() {
+ return zkConnectionString;
+ }
+
+ public void setZkConnectionString(String value) {
+ zkConnectionString = value;
+ }
+
+ public EventHubSpoutConfig withZkConnectionString(String value) {
+ setZkConnectionString(value);
+ return this;
+ }
+
+ public int getCheckpointIntervalInSeconds() {
+ return checkpointIntervalInSeconds;
+ }
+
+ public void setCheckpointIntervalInSeconds(int value) {
+ checkpointIntervalInSeconds = value;
+ }
+
+ public EventHubSpoutConfig withCheckpointIntervalInSeconds(int value) {
+ setCheckpointIntervalInSeconds(value);
+ return this;
+ }
+
+ public int getReceiverCredits() {
+ return receiverCredits;
+ }
+
+ public void setReceiverCredits(int value) {
+ receiverCredits = value;
+ }
+
+ public EventHubSpoutConfig withReceiverCredits(int value) {
+ setReceiverCredits(value);
+ return this;
+ }
+
+ public int getMaxPendingMsgsPerPartition() {
+ return maxPendingMsgsPerPartition;
+ }
+
+ public void setMaxPendingMsgsPerPartition(int value) {
+ maxPendingMsgsPerPartition = value;
+ }
+
+ public EventHubSpoutConfig withMaxPendingMsgsPerPartition(int value) {
+ setMaxPendingMsgsPerPartition(value);
+ return this;
+ }
+
+ public long getEnqueueTimeFilter() {
+ return enqueueTimeFilter;
+ }
+
+ public void setEnqueueTimeFilter(long value) {
+ enqueueTimeFilter = value;
+ }
+
+ public EventHubSpoutConfig withEnqueueTimeFilter(long value) {
+ setEnqueueTimeFilter(value);
+ return this;
+ }
+
+ public String getTopologyName() {
+ return topologyName;
+ }
+
+ public void setTopologyName(String value) {
+ topologyName = value;
+ }
+
+ public EventHubSpoutConfig withTopologyName(String value) {
+ setTopologyName(value);
+ return this;
+ }
+
+ public IEventDataScheme getEventDataScheme() {
+ return scheme;
+ }
+
+ public void setEventDataScheme(IEventDataScheme scheme) {
+ this.scheme = scheme;
+ }
+
+ public EventHubSpoutConfig withEventDataScheme(IEventDataScheme value) {
+ setEventDataScheme(value);
+ return this;
+ }
+
+ public String getConsumerGroupName() {
+ return consumerGroupName;
+ }
+
+ public void setConsumerGroupName(String value) {
+ consumerGroupName = value;
+ }
+
+ public EventHubSpoutConfig withConsumerGroupName(String value) {
+ setConsumerGroupName(value);
+ return this;
+ }
- public List<String> getPartitionList() {
- List<String> partitionList = new ArrayList<String>();
-
- for (int i = 0; i < this.partitionCount; i++) {
- partitionList.add(Integer.toString(i));
- }
-
- return partitionList;
- }
-
- public String getConnectionString() {
- return connectionString;
- }
-
- /*Keeping it for backward compatibility*/
- public void setTargetAddress(String targetFqnAddress) {
- }
-
- public void setTargetAddress(){
+ public List<String> getPartitionList() {
+ List<String> partitionList = new ArrayList<String>();
+
+ for (int i = 0; i < this.partitionCount; i++) {
+ partitionList.add(Integer.toString(i));
+ }
+
+ return partitionList;
+ }
+
+ public String getConnectionString() {
+ return connectionString;
+ }
+
+ /*Keeping it for backward compatibility*/
+ public void setTargetAddress(String targetFqnAddress) {
+ }
+
+ public void setTargetAddress() {
- }
+ }
- public EventHubSpoutConfig withTargetAddress(String targetFqnAddress) {
- setTargetAddress(targetFqnAddress);
- return this;
- }
-
- public String getOutputStreamId() {
- return outputStreamId;
- }
-
- public void setOutputStreamId(String outputStreamId) {
- this.outputStreamId = outputStreamId;
- }
+ public EventHubSpoutConfig withTargetAddress(String targetFqnAddress) {
+ setTargetAddress(targetFqnAddress);
+ return this;
+ }
+
+ public String getOutputStreamId() {
+ return outputStreamId;
+ }
+
+ public void setOutputStreamId(String outputStreamId) {
+ this.outputStreamId = outputStreamId;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/FieldConstants.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/FieldConstants.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/FieldConstants.java
index 88855eb..1baf16f 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/FieldConstants.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/FieldConstants.java
@@ -15,14 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
public class FieldConstants {
- public static final String PartitionKey = "partitionKey";
- public static final String Offset = "offset";
- public static final String Message = "message";
- public static final String META_DATA = "metadata";
- public static final String SYSTEM_META_DATA = "eventdata_system_properties";
- public static final String DefaultStartingOffset = "-1";
+ public static final String PartitionKey = "partitionKey";
+ public static final String Offset = "offset";
+ public static final String Message = "message";
+ public static final String META_DATA = "metadata";
+ public static final String SYSTEM_META_DATA = "eventdata_system_properties";
+ public static final String DefaultStartingOffset = "-1";
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventDataScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventDataScheme.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventDataScheme.java
index 854da6f..f2aa158 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventDataScheme.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventDataScheme.java
@@ -15,30 +15,30 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import com.microsoft.azure.eventhubs.EventData;
-import org.apache.storm.tuple.Fields;
-
import java.io.Serializable;
import java.util.List;
+import org.apache.storm.tuple.Fields;
public interface IEventDataScheme extends Serializable {
- /**
- * Deserialize an AMQP Message into a Tuple.
- *
- * @see #getOutputFields() for the list of fields the tuple will contain.
- *
- * @param eventData The EventData to Deserialize.
- * @return A tuple containing the deserialized fields of the message.
- */
- List<Object> deserialize(EventData eventData);
+ /**
+ * Deserialize an AMQP Message into a Tuple.
+ *
+ * @param eventData The EventData to Deserialize.
+ * @return A tuple containing the deserialized fields of the message.
+ *
+ * @see #getOutputFields() for the list of fields the tuple will contain.
+ */
+ List<Object> deserialize(EventData eventData);
- /**
- * Retrieve the Fields that are present on tuples created by this object.
- *
- * @return The Fields that are present on tuples created by this object.
- */
- Fields getOutputFields();
+ /**
+ * Retrieve the Fields that are present on tuples created by this object.
+ *
+ * @return The Fields that are present on tuples created by this object.
+ */
+ Fields getOutputFields();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventHubReceiver.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventHubReceiver.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventHubReceiver.java
index c8de8bc..4af967f 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventHubReceiver.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventHubReceiver.java
@@ -15,19 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import java.util.Map;
public interface IEventHubReceiver {
- void open(IEventFilter filter) throws EventHubException;
+ void open(IEventFilter filter) throws EventHubException;
- void close();
+ void close();
- boolean isOpen();
+ boolean isOpen();
- EventDataWrap receive();
+ EventDataWrap receive();
- Map<String, Object> getMetricsData();
+ Map<String, Object> getMetricsData();
}
[06/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/GenericResourceAwareStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/GenericResourceAwareStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/GenericResourceAwareStrategy.java
index 2b0ca40..af9c681 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/GenericResourceAwareStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/GenericResourceAwareStrategy.java
@@ -36,16 +36,51 @@ import org.slf4j.LoggerFactory;
public class GenericResourceAwareStrategy extends BaseResourceAwareStrategy implements IStrategy {
private static final Logger LOG = LoggerFactory.getLogger(GenericResourceAwareStrategy.class);
+ /**
+ * Implementation of the sortObjectResources method so other strategies can reuse it.
+ */
+ public static TreeSet<ObjectResources> sortObjectResourcesImpl(
+ final AllResources allResources, ExecutorDetails exec, TopologyDetails topologyDetails,
+ final ExistingScheduleFunc existingScheduleFunc) {
+ AllResources affinityBasedAllResources = new AllResources(allResources);
+
+ TreeSet<ObjectResources> sortedObjectResources =
+ new TreeSet<>((o1, o2) -> {
+ int execsScheduled1 = existingScheduleFunc.getNumExistingSchedule(o1.id);
+ int execsScheduled2 = existingScheduleFunc.getNumExistingSchedule(o2.id);
+ if (execsScheduled1 > execsScheduled2) {
+ return -1;
+ } else if (execsScheduled1 < execsScheduled2) {
+ return 1;
+ } else {
+ double o1Avg = allResources.availableResourcesOverall.calculateAveragePercentageUsedBy(o1.availableResources);
+ double o2Avg = allResources.availableResourcesOverall.calculateAveragePercentageUsedBy(o2.availableResources);
+
+ if (o1Avg > o2Avg) {
+ return -1;
+ } else if (o1Avg < o2Avg) {
+ return 1;
+ } else {
+ return o1.id.compareTo(o2.id);
+ }
+
+ }
+ });
+ sortedObjectResources.addAll(affinityBasedAllResources.objectResources);
+ LOG.debug("Sorted Object Resources: {}", sortedObjectResources);
+ return sortedObjectResources;
+ }
+
@Override
public SchedulingResult schedule(Cluster cluster, TopologyDetails td) {
prepare(cluster);
if (nodes.getNodes().size() <= 0) {
LOG.warn("No available nodes to schedule tasks on!");
return SchedulingResult.failure(
- SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES, "No available nodes to schedule tasks on!");
+ SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES, "No available nodes to schedule tasks on!");
}
Collection<ExecutorDetails> unassignedExecutors =
- new HashSet<>(this.cluster.getUnassignedExecutors(td));
+ new HashSet<>(this.cluster.getUnassignedExecutors(td));
LOG.info("ExecutorsNeedScheduling: {}", unassignedExecutors);
Collection<ExecutorDetails> scheduledTasks = new ArrayList<>();
List<Component> spouts = this.getSpouts(td);
@@ -53,7 +88,7 @@ public class GenericResourceAwareStrategy extends BaseResourceAwareStrategy impl
if (spouts.size() == 0) {
LOG.error("Cannot find a Spout!");
return SchedulingResult.failure(
- SchedulingStatus.FAIL_INVALID_TOPOLOGY, "Cannot find a Spout!");
+ SchedulingStatus.FAIL_INVALID_TOPOLOGY, "Cannot find a Spout!");
}
//order executors to be scheduled
@@ -64,10 +99,10 @@ public class GenericResourceAwareStrategy extends BaseResourceAwareStrategy impl
for (ExecutorDetails exec : orderedExecutors) {
LOG.debug(
- "Attempting to schedule: {} of component {}[ REQ {} ]",
- exec,
- td.getExecutorToComponent().get(exec),
- td.getTaskResourceReqList(exec));
+ "Attempting to schedule: {} of component {}[ REQ {} ]",
+ exec,
+ td.getExecutorToComponent().get(exec),
+ td.getTaskResourceReqList(exec));
final List<ObjectResources> sortedNodes = this.sortAllNodes(td, exec, favoredNodes, unFavoredNodes);
scheduleExecutor(exec, td, scheduledTasks, sortedNodes);
@@ -86,12 +121,12 @@ public class GenericResourceAwareStrategy extends BaseResourceAwareStrategy impl
if (executorsNotScheduled.size() > 0) {
LOG.error("Not all executors successfully scheduled: {}", executorsNotScheduled);
result =
- SchedulingResult.failure(
- SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES,
- (td.getExecutors().size() - unassignedExecutors.size())
- + "/"
- + td.getExecutors().size()
- + " executors scheduled");
+ SchedulingResult.failure(
+ SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES,
+ (td.getExecutors().size() - unassignedExecutors.size())
+ + "/"
+ + td.getExecutors().size()
+ + " executors scheduled");
} else {
LOG.debug("All resources successfully scheduled!");
result = SchedulingResult.success("Fully Scheduled by " + this.getClass().getSimpleName());
@@ -99,68 +134,28 @@ public class GenericResourceAwareStrategy extends BaseResourceAwareStrategy impl
return result;
}
-
/**
- * Sort objects by the following two criteria. 1) the number executors of the topology that needs
- * to be scheduled is already on the object (node or rack) in descending order. The reasoning to
- * sort based on criterion 1 is so we schedule the rest of a topology on the same object (node or
- * rack) as the existing executors of the topology. 2) the subordinate/subservient resource
- * availability percentage of a rack in descending order We calculate the resource availability
- * percentage by dividing the resource availability of the object (node or rack) by the resource
- * availability of the entire rack or cluster depending on if object references a node or a rack.
- * How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node or rack
- * if it is requested by the executor that the sorting is being done for and pulls it down if it is not.
- * By doing this calculation, objects (node or rack) that have exhausted or little of one of the
- * resources mentioned above will be ranked after racks that have more balanced resource
- * availability and nodes or racks that have resources that are not requested will be ranked below
- * . So we will be less likely to pick a rack that have a lot of one resource but a
- * low amount of another and have a lot of resources that are not requested by the executor.
+ * Sort objects by the following two criteria. 1) the number executors of the topology that needs to be scheduled is already on the
+ * object (node or rack) in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the
+ * same object (node or rack) as the existing executors of the topology. 2) the subordinate/subservient resource availability percentage
+ * of a rack in descending order We calculate the resource availability percentage by dividing the resource availability of the object
+ * (node or rack) by the resource availability of the entire rack or cluster depending on if object references a node or a rack. How
+ * this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node or rack if it is requested by the executor
+ * that the sorting is being done for and pulls it down if it is not. By doing this calculation, objects (node or rack) that have
+ * exhausted or little of one of the resources mentioned above will be ranked after racks that have more balanced resource availability
+ * and nodes or racks that have resources that are not requested will be ranked below . So we will be less likely to pick a rack that
+ * have a lot of one resource but a low amount of another and have a lot of resources that are not requested by the executor.
*
- * @param allResources contains all individual ObjectResources as well as cumulative stats
- * @param exec executor for which the sorting is done
- * @param topologyDetails topologyDetails for the above executor
+ * @param allResources contains all individual ObjectResources as well as cumulative stats
+ * @param exec executor for which the sorting is done
+ * @param topologyDetails topologyDetails for the above executor
* @param existingScheduleFunc a function to get existing executors already scheduled on this object
* @return a sorted list of ObjectResources
*/
@Override
protected TreeSet<ObjectResources> sortObjectResources(
- final AllResources allResources, ExecutorDetails exec, TopologyDetails topologyDetails,
- final ExistingScheduleFunc existingScheduleFunc) {
- return sortObjectResourcesImpl(allResources, exec, topologyDetails, existingScheduleFunc);
- }
-
- /**
- * Implementation of the sortObjectResources method so other strategies can reuse it.
- */
- public static TreeSet<ObjectResources> sortObjectResourcesImpl(
final AllResources allResources, ExecutorDetails exec, TopologyDetails topologyDetails,
final ExistingScheduleFunc existingScheduleFunc) {
- AllResources affinityBasedAllResources = new AllResources(allResources);
-
- TreeSet<ObjectResources> sortedObjectResources =
- new TreeSet<>((o1, o2) -> {
- int execsScheduled1 = existingScheduleFunc.getNumExistingSchedule(o1.id);
- int execsScheduled2 = existingScheduleFunc.getNumExistingSchedule(o2.id);
- if (execsScheduled1 > execsScheduled2) {
- return -1;
- } else if (execsScheduled1 < execsScheduled2) {
- return 1;
- } else {
- double o1Avg = allResources.availableResourcesOverall.calculateAveragePercentageUsedBy(o1.availableResources);
- double o2Avg = allResources.availableResourcesOverall.calculateAveragePercentageUsedBy(o2.availableResources);
-
- if (o1Avg > o2Avg) {
- return -1;
- } else if (o1Avg < o2Avg) {
- return 1;
- } else {
- return o1.id.compareTo(o2.id);
- }
-
- }
- });
- sortedObjectResources.addAll(affinityBasedAllResources.objectResources);
- LOG.debug("Sorted Object Resources: {}", sortedObjectResources);
- return sortedObjectResources;
+ return sortObjectResourcesImpl(allResources, exec, topologyDetails, existingScheduleFunc);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/IStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/IStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/IStrategy.java
index 53859fd..2cdbb32 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/IStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/IStrategy.java
@@ -1,25 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource.strategies.scheduling;
import java.util.Map;
-
import org.apache.storm.scheduler.Cluster;
import org.apache.storm.scheduler.TopologyDetails;
import org.apache.storm.scheduler.resource.SchedulingResult;
@@ -29,13 +22,13 @@ import org.apache.storm.scheduler.resource.SchedulingResult;
* In the future strategies will be pluggable
*/
public interface IStrategy {
-
+
/**
* Prepare the Strategy for scheduling.
* @param config the cluster configuration
*/
void prepare(Map<String, Object> config);
-
+
/**
* This method is invoked to calculate a scheduling for topology td. Cluster will reject any changes that are
* not for the given topology. Any changes made to the cluster will be committed if the scheduling is successful.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
index 5d3dfbe..cfb6bb7 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.utils;
@@ -35,9 +29,7 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
-import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
-import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.ServerConfigUtils;
import org.apache.storm.utils.Time;
import org.apache.storm.utils.Utils;
@@ -84,7 +76,7 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
}
Integer thisPollTime = (Integer) conf.get(DaemonConfig.SCHEDULER_CONFIG_LOADER_POLLTIME_SECS);
if (thisPollTime != null) {
- artifactoryPollTimeSecs =thisPollTime;
+ artifactoryPollTimeSecs = thisPollTime;
}
String thisBase = (String) conf.get(DaemonConfig.SCHEDULER_CONFIG_LOADER_ARTIFACTORY_BASE_DIRECTORY);
if (thisBase != null) {
@@ -97,7 +89,7 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
try {
targetURI = new URI(uriString);
scheme = targetURI.getScheme().substring(ARTIFACTORY_SCHEME_PREFIX.length());
- } catch(URISyntaxException e) {
+ } catch (URISyntaxException e) {
LOG.error("Failed to parse uri={}", uriString);
}
}
@@ -119,7 +111,7 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
int currentTimeSecs = Time.currentTimeSecs();
if (lastReturnedValue != null && ((currentTimeSecs - lastReturnedTime) < artifactoryPollTimeSecs)) {
LOG.debug("currentTimeSecs: {}; lastReturnedTime {}; artifactoryPollTimeSecs: {}. Returning our last map.",
- currentTimeSecs, lastReturnedTime, artifactoryPollTimeSecs);
+ currentTimeSecs, lastReturnedTime, artifactoryPollTimeSecs);
return (Map<String, Object>) lastReturnedValue.get(configKey);
}
@@ -134,41 +126,6 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
return null;
}
-
- /**
- * A private class used to check the response coming back from httpclient.
- */
- private static class GETStringResponseHandler implements ResponseHandler<String> {
- private static GETStringResponseHandler singleton = null;
-
- /**
- * @return a singleton httpclient GET response handler
- */
- public static GETStringResponseHandler getInstance() {
- if (singleton == null) {
- singleton = new GETStringResponseHandler();
- }
- return singleton;
- }
-
- /**
- * @param response The http response to verify.
- * @return null on failure or the response string if return code is in 200 range
- */
- @Override
- public String handleResponse(final HttpResponse response) throws IOException {
- int status = response.getStatusLine().getStatusCode();
- HttpEntity entity = response.getEntity();
- String entityString = (entity != null ? EntityUtils.toString(entity) : null);
- if (status >= 200 && status < 300) {
- return entityString;
- } else {
- LOG.error("Got unexpected response code {}; entity: {}", status, entityString);
- return null;
- }
- }
- }
-
/**
* @param api null if we are trying to download artifact, otherwise a string to call REST api,
* e.g. "/api/storage"
@@ -232,13 +189,6 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
return returnValue;
}
- private class DirEntryCompare implements Comparator<JSONObject> {
- @Override
- public int compare(JSONObject o1, JSONObject o2) {
- return ((String)o1.get("uri")).compareTo((String)o2.get("uri"));
- }
- }
-
private String loadMostRecentArtifact(String location, String host, Integer port) {
// Is this a directory or is it a file?
JSONObject json = getArtifactMetadata(location, host, port);
@@ -256,7 +206,7 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
return returnValue;
}
- // This should mean that we were pointed at a directory.
+ // This should mean that we were pointed at a directory.
// Find the most recent child and load that.
JSONArray msg = (JSONArray) json.get("children");
if (msg == null || msg.size() == 0) {
@@ -336,13 +286,13 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
// First make the cache dir
String localDirName = ServerConfigUtils.masterLocalDir(conf) + File.separator + LOCAL_ARTIFACT_DIR;
File dir = new File(localDirName);
- if (! dir.exists()) {
+ if (!dir.exists()) {
dir.mkdirs();
}
localCacheDir = localDirName + File.separator + location.replaceAll(File.separator, "_");
dir = new File(localCacheDir);
- if (! dir.exists()) {
+ if (!dir.exists()) {
dir.mkdir();
}
cacheInitialized = true;
@@ -388,4 +338,45 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
return null;
}
+
+ /**
+ * A private class used to check the response coming back from httpclient.
+ */
+ private static class GETStringResponseHandler implements ResponseHandler<String> {
+ private static GETStringResponseHandler singleton = null;
+
+ /**
+ * @return a singleton httpclient GET response handler
+ */
+ public static GETStringResponseHandler getInstance() {
+ if (singleton == null) {
+ singleton = new GETStringResponseHandler();
+ }
+ return singleton;
+ }
+
+ /**
+ * @param response The http response to verify.
+ * @return null on failure or the response string if return code is in 200 range
+ */
+ @Override
+ public String handleResponse(final HttpResponse response) throws IOException {
+ int status = response.getStatusLine().getStatusCode();
+ HttpEntity entity = response.getEntity();
+ String entityString = (entity != null ? EntityUtils.toString(entity) : null);
+ if (status >= 200 && status < 300) {
+ return entityString;
+ } else {
+ LOG.error("Got unexpected response code {}; entity: {}", status, entityString);
+ return null;
+ }
+ }
+ }
+
+ private class DirEntryCompare implements Comparator<JSONObject> {
+ @Override
+ public int compare(JSONObject o1, JSONObject o2) {
+ return ((String) o1.get("uri")).compareTo((String) o2.get("uri"));
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderFactory.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderFactory.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderFactory.java
index 6494ae2..71b9eb5 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderFactory.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoaderFactory.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.utils;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/utils/ConfigLoaderFactoryService.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ConfigLoaderFactoryService.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ConfigLoaderFactoryService.java
index 53df78b..4e1b3a5 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ConfigLoaderFactoryService.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ConfigLoaderFactoryService.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.utils;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoader.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoader.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoader.java
index a75e92b..8e4c9b9 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoader.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoader.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.utils;
@@ -21,7 +15,6 @@ package org.apache.storm.scheduler.utils;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Map;
-
import org.apache.storm.DaemonConfig;
import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoaderFactory.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoaderFactory.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoaderFactory.java
index 9a2ac5c..c9abd1b 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoaderFactory.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/FileConfigLoaderFactory.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.utils;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoader.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoader.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoader.java
index 5071133..6bfd6be 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoader.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoader.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.utils;
@@ -27,6 +21,6 @@ public interface IConfigLoader {
* @param configKey The key from which we want to get the scheduler config.
* @return The scheduler configs
*/
- Map<?,?> load(String configKey);
+ Map<?, ?> load(String configKey);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoaderFactory.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoaderFactory.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoaderFactory.java
index a039870..115c050 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoaderFactory.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/IConfigLoaderFactory.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.utils;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/security/auth/workertoken/WorkerTokenManager.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/security/auth/workertoken/WorkerTokenManager.java b/storm-server/src/main/java/org/apache/storm/security/auth/workertoken/WorkerTokenManager.java
index dac0c91..8997e6f 100644
--- a/storm-server/src/main/java/org/apache/storm/security/auth/workertoken/WorkerTokenManager.java
+++ b/storm-server/src/main/java/org/apache/storm/security/auth/workertoken/WorkerTokenManager.java
@@ -43,44 +43,21 @@ public class WorkerTokenManager {
private static final Logger LOG = LoggerFactory.getLogger(WorkerTokenManager.class);
/**
- * The length of the random keys to use in bits.
- * This should be at least the length of WorkerTokenSigner.DEFAULT_HMAC_ALGORITHM.
+ * The length of the random keys to use in bits. This should be at least the length of WorkerTokenSigner.DEFAULT_HMAC_ALGORITHM.
*/
private static final int KEY_LENGTH = 256;
-
- /**
- * Generate a new random secret key.
- * @return the new key
- */
- protected SecretKey generateSecret() {
- SecretKey key;
- synchronized (keyGen) {
- key = keyGen.generateKey();
- }
- return key;
- }
-
- /**
- * Get the secret that should be used to sign a token. This may either reuse a secret or generate a new one so any user should
- * call this once and save the result.
- * @return the key to use.
- */
- protected SecretKey getCurrentSecret() {
- return generateSecret();
- }
-
/**
* Key generator to use.
*/
private final KeyGenerator keyGen;
private final IStormClusterState state;
private final long tokenLifetimeMillis;
-
/**
- * Constructor. This assumes that state can store the tokens securely, and that they should be enabled at all.
- * Please use AuthUtils.areWorkerTokensEnabledServer to validate this first.
+ * Constructor. This assumes that state can store the tokens securely, and that they should be enabled at all. Please use
+ * AuthUtils.areWorkerTokensEnabledServer to validate this first.
+ *
* @param daemonConf the config for nimbus.
- * @param state the state used to store private keys.
+ * @param state the state used to store private keys.
*/
public WorkerTokenManager(Map<String, Object> daemonConf, IStormClusterState state) {
this.state = state;
@@ -91,15 +68,39 @@ public class WorkerTokenManager {
throw new IllegalArgumentException("Can't find " + WorkerTokenSigner.DEFAULT_HMAC_ALGORITHM + " algorithm.");
}
this.tokenLifetimeMillis = TimeUnit.MILLISECONDS.convert(
- ObjectReader.getLong(daemonConf.get(DaemonConfig.STORM_WORKER_TOKEN_LIFE_TIME_HOURS),24L),
+ ObjectReader.getLong(daemonConf.get(DaemonConfig.STORM_WORKER_TOKEN_LIFE_TIME_HOURS), 24L),
TimeUnit.HOURS);
}
/**
+ * Generate a new random secret key.
+ *
+ * @return the new key
+ */
+ protected SecretKey generateSecret() {
+ SecretKey key;
+ synchronized (keyGen) {
+ key = keyGen.generateKey();
+ }
+ return key;
+ }
+
+ /**
+ * Get the secret that should be used to sign a token. This may either reuse a secret or generate a new one so any user should call
+ * this once and save the result.
+ *
+ * @return the key to use.
+ */
+ protected SecretKey getCurrentSecret() {
+ return generateSecret();
+ }
+
+ /**
* Create or update an existing key.
+ *
* @param serviceType the type of service to create a token for
- * @param user the user the token is for
- * @param topologyId the topology the token is for
+ * @param user the user the token is for
+ * @param topologyId the topology the token is for
* @return a newly generated token that should be good to start using form now until it expires.
*/
public WorkerToken createOrUpdateTokenFor(WorkerTokenServiceType serviceType, String user, String topologyId) {
@@ -118,6 +119,7 @@ public class WorkerTokenManager {
/**
* Get the maximum expiration token time that should be renewed.
+ *
* @return any token with an expiration less than the returned value should be renewed.
*/
public long getMaxExpirationTimeForRenewal() {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/testing/CompleteTopologyParam.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/testing/CompleteTopologyParam.java b/storm-server/src/main/java/org/apache/storm/testing/CompleteTopologyParam.java
index 861e21a..eafb315 100644
--- a/storm-server/src/main/java/org/apache/storm/testing/CompleteTopologyParam.java
+++ b/storm-server/src/main/java/org/apache/storm/testing/CompleteTopologyParam.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.testing;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.Testing;
@@ -26,78 +20,78 @@ import org.apache.storm.Testing;
* The param class for the `Testing.completeTopology`.
*/
public class CompleteTopologyParam {
- /**
- * The mocked spout sources
- */
- private MockedSources mockedSources = new MockedSources();
- /**
- * the config for the topology when it was submitted to the cluster
- */
- private Map<String, Object> topoConf = new Config();
- /**
- * whether cleanup the state?
- */
- private boolean cleanupState = true;
- /**
- * the topology name you want to submit to the cluster
- */
- private String topologyName;
+ /**
+ * The mocked spout sources
+ */
+ private MockedSources mockedSources = new MockedSources();
+ /**
+ * the config for the topology when it was submitted to the cluster
+ */
+ private Map<String, Object> topoConf = new Config();
+ /**
+ * whether cleanup the state?
+ */
+ private boolean cleanupState = true;
+ /**
+ * the topology name you want to submit to the cluster
+ */
+ private String topologyName;
+
+ /**
+ * the timeout of topology you want to submit to the cluster
+ */
+ private int timeoutMs = Testing.TEST_TIMEOUT_MS;
- /**
- * the timeout of topology you want to submit to the cluster
- */
- private int timeoutMs = Testing.TEST_TIMEOUT_MS;
+ public MockedSources getMockedSources() {
+ return mockedSources;
+ }
- public MockedSources getMockedSources() {
- return mockedSources;
- }
+ public void setMockedSources(MockedSources mockedSources) {
+ if (mockedSources == null) {
+ mockedSources = new MockedSources();
+ }
- public void setMockedSources(MockedSources mockedSources) {
- if (mockedSources == null) {
- mockedSources = new MockedSources();
- }
-
- this.mockedSources = mockedSources;
- }
+ this.mockedSources = mockedSources;
+ }
- public Map<String, Object> getStormConf() {
- return topoConf;
- }
+ public Map<String, Object> getStormConf() {
+ return topoConf;
+ }
- public void setStormConf(Map<String, Object> topoConf) {
- if (topoConf == null) {
- topoConf = new Config();
- }
- this.topoConf = topoConf;
- }
+ public void setStormConf(Map<String, Object> topoConf) {
+ if (topoConf == null) {
+ topoConf = new Config();
+ }
+ this.topoConf = topoConf;
+ }
- public boolean getCleanupState() {
- return cleanupState;
- }
+ public boolean getCleanupState() {
+ return cleanupState;
+ }
- public void setCleanupState(Boolean cleanupState) {
- if (cleanupState == null) {
- cleanupState = true;
- }
- this.cleanupState = cleanupState;
- }
+ public void setCleanupState(Boolean cleanupState) {
+ if (cleanupState == null) {
+ cleanupState = true;
+ }
+ this.cleanupState = cleanupState;
+ }
- public String getTopologyName() {
- return topologyName;
- }
+ public String getTopologyName() {
+ return topologyName;
+ }
- public void setTopologyName(String topologyName) {
- this.topologyName = topologyName;
- }
+ public void setTopologyName(String topologyName) {
+ this.topologyName = topologyName;
+ }
- public Integer getTimeoutMs() {
- return timeoutMs;
- }
+ public Integer getTimeoutMs() {
+ return timeoutMs;
+ }
- public void setTimeoutMs(Integer timeoutMs) {
- if (timeoutMs == null) {
- timeoutMs = Testing.TEST_TIMEOUT_MS;
- }
- this.timeoutMs = timeoutMs;
- }
+ public void setTimeoutMs(Integer timeoutMs) {
+ if (timeoutMs == null) {
+ timeoutMs = Testing.TEST_TIMEOUT_MS;
+ }
+ this.timeoutMs = timeoutMs;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/testing/InProcessZookeeper.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/testing/InProcessZookeeper.java b/storm-server/src/main/java/org/apache/storm/testing/InProcessZookeeper.java
index 032f09c..38db6da 100644
--- a/storm-server/src/main/java/org/apache/storm/testing/InProcessZookeeper.java
+++ b/storm-server/src/main/java/org/apache/storm/testing/InProcessZookeeper.java
@@ -1,25 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.testing;
import java.util.List;
-
import org.apache.storm.zookeeper.Zookeeper;
import org.apache.zookeeper.server.NIOServerCnxnFactory;
@@ -30,11 +23,11 @@ import org.apache.zookeeper.server.NIOServerCnxnFactory;
* }
*/
public class InProcessZookeeper implements AutoCloseable {
-
+
private final TmpPath zkTmp;
private final NIOServerCnxnFactory zookeeper;
private final long zkPort;
-
+
public InProcessZookeeper() throws Exception {
zkTmp = new TmpPath();
@SuppressWarnings("unchecked")
@@ -42,7 +35,7 @@ public class InProcessZookeeper implements AutoCloseable {
zkPort = (Long) portAndHandle.get(0);
zookeeper = (NIOServerCnxnFactory) portAndHandle.get(1);
}
-
+
/**
* @return the port ZK is listening on (localhost)
*/
@@ -55,5 +48,5 @@ public class InProcessZookeeper implements AutoCloseable {
Zookeeper.shutdownInprocessZookeeper(zookeeper);
zkTmp.close();
}
-
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/testing/TestJob.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/testing/TestJob.java b/storm-server/src/main/java/org/apache/storm/testing/TestJob.java
index 79288ad..9c118e8 100644
--- a/storm-server/src/main/java/org/apache/storm/testing/TestJob.java
+++ b/storm-server/src/main/java/org/apache/storm/testing/TestJob.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.testing;
import org.apache.storm.ILocalCluster;
@@ -33,11 +28,11 @@ import org.apache.storm.ILocalCluster;
* ```
*/
public interface TestJob {
- /**
- * run the testing logic with the cluster.
- *
- * @param cluster the cluster which created by <code>Testing.withSimulatedTimeLocalCluster</code>
- * and <code>Testing.withTrackedCluster</code>.
- */
+ /**
+ * run the testing logic with the cluster.
+ *
+ * @param cluster the cluster which created by <code>Testing.withSimulatedTimeLocalCluster</code>
+ * and <code>Testing.withTrackedCluster</code>.
+ */
public void run(ILocalCluster cluster) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/testing/TrackedTopology.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/testing/TrackedTopology.java b/storm-server/src/main/java/org/apache/storm/testing/TrackedTopology.java
index 00bd1e7..1c6f738 100644
--- a/storm-server/src/main/java/org/apache/storm/testing/TrackedTopology.java
+++ b/storm-server/src/main/java/org/apache/storm/testing/TrackedTopology.java
@@ -1,29 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.testing;
-import static org.apache.storm.Testing.whileTimeout;
+package org.apache.storm.testing;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicInteger;
-
import org.apache.storm.ILocalCluster;
import org.apache.storm.Testing;
import org.apache.storm.Thrift;
@@ -37,6 +29,8 @@ import org.apache.storm.utils.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.storm.Testing.whileTimeout;
+
/**
* A tracked topology keeps metrics for every bolt and spout.
* This allows a test to know how many tuples have been fully processed.
@@ -48,88 +42,88 @@ public class TrackedTopology {
private final StormTopology topology;
private final AtomicInteger lastSpoutCommit;
private final ILocalCluster cluster;
-
+
/**
* Create a new topology to be tracked.
* @param origTopo the original topology.
* @param cluster a cluster that should have been launched with tracking enabled.
*/
- public TrackedTopology(StormTopology origTopo, ILocalCluster cluster) {
- LOG.warn("CLUSTER {} - {}", cluster, cluster.getTrackedId());
- this.cluster = cluster;
- lastSpoutCommit = new AtomicInteger(0);
- String id = cluster.getTrackedId();
- topology = origTopo.deepCopy();
- for (Bolt bolt: topology.get_bolts().values()) {
- IRichBolt obj = (IRichBolt) Thrift.deserializeComponentObject(bolt.get_bolt_object());
- bolt.set_bolt_object(Thrift.serializeComponentObject(new BoltTracker(obj, id)));
- }
- for (SpoutSpec spout: topology.get_spouts().values()) {
- IRichSpout obj = (IRichSpout) Thrift.deserializeComponentObject(spout.get_spout_object());
- spout.set_spout_object(Thrift.serializeComponentObject(new SpoutTracker(obj, id)));
- }
- }
+ public TrackedTopology(StormTopology origTopo, ILocalCluster cluster) {
+ LOG.warn("CLUSTER {} - {}", cluster, cluster.getTrackedId());
+ this.cluster = cluster;
+ lastSpoutCommit = new AtomicInteger(0);
+ String id = cluster.getTrackedId();
+ topology = origTopo.deepCopy();
+ for (Bolt bolt : topology.get_bolts().values()) {
+ IRichBolt obj = (IRichBolt) Thrift.deserializeComponentObject(bolt.get_bolt_object());
+ bolt.set_bolt_object(Thrift.serializeComponentObject(new BoltTracker(obj, id)));
+ }
+ for (SpoutSpec spout : topology.get_spouts().values()) {
+ IRichSpout obj = (IRichSpout) Thrift.deserializeComponentObject(spout.get_spout_object());
+ spout.set_spout_object(Thrift.serializeComponentObject(new SpoutTracker(obj, id)));
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static int globalAmt(String id, String key) {
+ LOG.warn("Reading tracked metrics for ID {}", id);
+ return ((ConcurrentHashMap<String, AtomicInteger>) RegisteredGlobalState.getState(id)).get(key).get();
+ }
public StormTopology getTopology() {
- return topology;
+ return topology;
+ }
+
+ public ILocalCluster getCluster() {
+ return cluster;
+ }
+
+ /**
+ * Wait for 1 tuple to be fully processed
+ */
+ public void trackedWait() {
+ trackedWait(1, Testing.TEST_TIMEOUT_MS);
+ }
+
+ /**
+ * Wait for amt tuples to be fully processed.
+ */
+ public void trackedWait(int amt) {
+ trackedWait(amt, Testing.TEST_TIMEOUT_MS);
+ }
+
+ /**
+ * Wait for amt tuples to be fully processed timeoutMs happens.
+ */
+ public void trackedWait(int amt, int timeoutMs) {
+ final int target = amt + lastSpoutCommit.get();
+ final String id = cluster.getTrackedId();
+ Random rand = ThreadLocalRandom.current();
+ whileTimeout(timeoutMs,
+ () -> {
+ int se = globalAmt(id, "spout-emitted");
+ int transferred = globalAmt(id, "transferred");
+ int processed = globalAmt(id, "processed");
+ LOG.info("emitted {} target {} transferred {} processed {}", se, target, transferred, processed);
+ return (target != se) || (transferred != processed);
+ },
+ () -> {
+ Time.advanceTimeSecs(1);
+ try {
+ Thread.sleep(rand.nextInt(200));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ lastSpoutCommit.set(target);
}
-
- public ILocalCluster getCluster() {
- return cluster;
- }
-
- /**
- * Wait for 1 tuple to be fully processed
- */
- public void trackedWait() {
- trackedWait(1, Testing.TEST_TIMEOUT_MS);
- }
-
- /**
- * Wait for amt tuples to be fully processed.
- */
- public void trackedWait(int amt) {
- trackedWait(amt, Testing.TEST_TIMEOUT_MS);
- }
-
- /**
- * Wait for amt tuples to be fully processed timeoutMs happens.
- */
- public void trackedWait(int amt, int timeoutMs) {
- final int target = amt + lastSpoutCommit.get();
- final String id = cluster.getTrackedId();
- Random rand = ThreadLocalRandom.current();
- whileTimeout(timeoutMs,
- () -> {
- int se = globalAmt(id, "spout-emitted");
- int transferred = globalAmt(id, "transferred");
- int processed = globalAmt(id, "processed");
- LOG.info("emitted {} target {} transferred {} processed {}", se, target, transferred, processed);
- return (target != se) || (transferred != processed);
- },
- () -> {
- Time.advanceTimeSecs(1);
- try {
- Thread.sleep(rand.nextInt(200));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- });
- lastSpoutCommit.set(target);
- }
- /**
+ /**
* Read a metric from the tracked cluster (NOT JUST THIS TOPOLOGY)
* @param key one of "spout-emitted", "processed", or "transferred"
* @return the amount of that metric
*/
- public int globalAmt(String key) {
- return globalAmt(cluster.getTrackedId(), key);
- }
-
- @SuppressWarnings("unchecked")
- private static int globalAmt(String id, String key) {
- LOG.warn("Reading tracked metrics for ID {}", id);
- return ((ConcurrentHashMap<String, AtomicInteger>)RegisteredGlobalState.getState(id)).get(key).get();
+ public int globalAmt(String key) {
+ return globalAmt(cluster.getTrackedId(), key);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/utils/BufferInputStream.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/utils/BufferInputStream.java b/storm-server/src/main/java/org/apache/storm/utils/BufferInputStream.java
index cd230c3..5bf91ca 100644
--- a/storm-server/src/main/java/org/apache/storm/utils/BufferInputStream.java
+++ b/storm-server/src/main/java/org/apache/storm/utils/BufferInputStream.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.utils;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/utils/LruMap.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/utils/LruMap.java b/storm-server/src/main/java/org/apache/storm/utils/LruMap.java
index 3ed5d06..ea0ea54 100644
--- a/storm-server/src/main/java/org/apache/storm/utils/LruMap.java
+++ b/storm-server/src/main/java/org/apache/storm/utils/LruMap.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.utils;
@@ -40,7 +34,7 @@ public class LruMap<A, B> extends LinkedHashMap<A, B> {
this(maxSize);
this.evCb = evictionCallback;
}
-
+
@Override
protected boolean removeEldestEntry(final Map.Entry<A, B> eldest) {
boolean evict = size() > this.maxSize;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java b/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
index dcf3d09..1df19cb 100644
--- a/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
@@ -27,13 +27,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
-
import org.apache.commons.io.FileUtils;
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
-
public class ServerConfigUtils {
public static final String FILE_SEPARATOR = File.separator;
public static final String NIMBUS_DO_NOT_REASSIGN = "NIMBUS-DO-NOT-REASSIGN";
@@ -44,9 +42,9 @@ public class ServerConfigUtils {
private static ServerConfigUtils _instance = new ServerConfigUtils();
/**
- * Provide an instance of this class for delegates to use. To mock out
- * delegated methods, provide an instance of a subclass that overrides the
- * implementation of the delegated method.
+ * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that
+ * overrides the implementation of the delegated method.
+ *
* @param u a ServerConfigUtils instance
* @return the previously set instance
*/
@@ -85,8 +83,8 @@ public class ServerConfigUtils {
/* TODO: make sure test these two functions in manual tests */
public static List<String> getTopoLogsUsers(Map<String, Object> topologyConf) {
- List<String> logsUsers = (List<String>)topologyConf.get(DaemonConfig.LOGS_USERS);
- List<String> topologyUsers = (List<String>)topologyConf.get(Config.TOPOLOGY_USERS);
+ List<String> logsUsers = (List<String>) topologyConf.get(DaemonConfig.LOGS_USERS);
+ List<String> topologyUsers = (List<String>) topologyConf.get(Config.TOPOLOGY_USERS);
Set<String> mergedUsers = new HashSet<String>();
if (logsUsers != null) {
for (String user : logsUsers) {
@@ -108,8 +106,8 @@ public class ServerConfigUtils {
}
public static List<String> getTopoLogsGroups(Map<String, Object> topologyConf) {
- List<String> logsGroups = (List<String>)topologyConf.get(DaemonConfig.LOGS_GROUPS);
- List<String> topologyGroups = (List<String>)topologyConf.get(Config.TOPOLOGY_GROUPS);
+ List<String> logsGroups = (List<String>) topologyConf.get(DaemonConfig.LOGS_GROUPS);
+ List<String> topologyGroups = (List<String>) topologyConf.get(Config.TOPOLOGY_GROUPS);
Set<String> mergedGroups = new HashSet<String>();
if (logsGroups != null) {
for (String group : logsGroups) {
[61/63] [abbrv] storm git commit: Merge branch 'master' of
github.com:apache/storm into fix-style
Posted by bo...@apache.org.
Merge branch 'master' of github.com:apache/storm into fix-style
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/99c62d49
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/99c62d49
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/99c62d49
Branch: refs/heads/master
Commit: 99c62d4943904233197f9d820ce63cacf16ec0f0
Parents: e8ffac4 7673253
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Wed Apr 25 16:58:32 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Apr 25 17:04:24 2018 -0400
----------------------------------------------------------------------
.../storm/cluster/PaceMakerStateStorage.java | 2 +-
.../apache/storm/pacemaker/PacemakerClient.java | 40 +++++++++++---------
.../storm/pacemaker/PacemakerClientPool.java | 33 ++++++++++++----
3 files changed, 50 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/99c62d49/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/99c62d49/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java
----------------------------------------------------------------------
diff --cc storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java
index d49db8b,8b46f55..1f03854
--- a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java
+++ b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java
@@@ -51,6 -56,10 +51,8 @@@ public class PacemakerClient implement
private HBMessage messages[];
private LinkedBlockingQueue<Integer> availableMessageSlots;
private ThriftNettyClientCodec.AuthMethod authMethod;
+ private static final int maxRetries = 10;
-
- private static Timer timer = new Timer(true);
-
++
private StormBoundedExponentialBackoffRetry backoff = new StormBoundedExponentialBackoffRetry(100, 5000, 20);
private int retryTimes = 0;
http://git-wip-us.apache.org/repos/asf/storm/blob/99c62d49/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java
----------------------------------------------------------------------
diff --cc storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java
index dacd2e0,539e0a8..6721ecb
--- a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java
+++ b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java
@@@ -54,12 -66,15 +59,15 @@@ public class PacemakerClientPool
public List<HBMessage> sendAll(HBMessage m) throws PacemakerConnectionException, InterruptedException {
List<HBMessage> responses = new ArrayList<HBMessage>();
LOG.debug("Using servers: {}", servers);
- for (String s : servers) {
- HBMessage response = getClientForServer(s).send(m);
- responses.add(response);
-
+ for(String s : servers) {
+ try {
+ HBMessage response = getClientForServer(s).send(m);
+ responses.add(response);
+ } catch (PacemakerConnectionException e) {
+ LOG.warn("Failed to connect to the pacemaker server {}", s);
+ }
}
- if(responses.size() == 0) {
+ if (responses.size() == 0) {
throw new PacemakerConnectionException("Failed to connect to any Pacemaker.");
}
return responses;
[28/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-hdfs
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java
index 6d626be..af0127a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java
@@ -1,12 +1,7 @@
-
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
@@ -28,77 +23,60 @@ import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.storm.Config;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.storm.Config;
import org.apache.storm.hdfs.common.HdfsUtils;
import org.apache.storm.hdfs.security.HdfsSecurityUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HdfsSpout extends BaseRichSpout {
+ // other members
+ private static final Logger LOG = LoggerFactory.getLogger(HdfsSpout.class);
+ private final AtomicBoolean commitTimeElapsed = new AtomicBoolean(false);
+ HashMap<MessageId, List<Object>> inflight = new HashMap<>();
+ LinkedBlockingQueue<HdfsUtils.Pair<MessageId, List<Object>>> retryList = new LinkedBlockingQueue<>();
+ HdfsUtils.Pair<Path, FileLock.LogEntry> lastExpiredLock = null;
// user configurable
private String hdfsUri; // required
private String readerType; // required
private Fields outputFields; // required
-
private String sourceDir; // required
private Path sourceDirPath; // required
-
private String archiveDir; // required
private Path archiveDirPath; // required
-
private String badFilesDir; // required
private Path badFilesDirPath; // required
-
private String lockDir;
private Path lockDirPath;
-
private int commitFrequencyCount = Configs.DEFAULT_COMMIT_FREQ_COUNT;
private int commitFrequencySec = Configs.DEFAULT_COMMIT_FREQ_SEC;
private int maxOutstanding = Configs.DEFAULT_MAX_OUTSTANDING;
private int lockTimeoutSec = Configs.DEFAULT_LOCK_TIMEOUT;
private boolean clocksInSync = true;
-
private String inprogress_suffix = ".inprogress"; // not configurable to prevent change between topology restarts
private String ignoreSuffix = ".ignore";
-
private String outputStreamName = null;
-
- // other members
- private static final Logger LOG = LoggerFactory.getLogger(HdfsSpout.class);
-
private ProgressTracker tracker = null;
-
private FileSystem hdfs;
private FileReader reader;
-
private SpoutOutputCollector collector;
- HashMap<MessageId, List<Object>> inflight = new HashMap<>();
- LinkedBlockingQueue<HdfsUtils.Pair<MessageId, List<Object>>> retryList = new LinkedBlockingQueue<>();
-
private Configuration hdfsConfig;
-
private Map<String, Object> conf = null;
private FileLock lock;
private String spoutId = null;
-
- HdfsUtils.Pair<Path, FileLock.LogEntry> lastExpiredLock = null;
private long lastExpiredLockTime = 0;
-
private long tupleCounter = 0;
private boolean ackEnabled = false;
private int acksSinceLastCommit = 0;
- private final AtomicBoolean commitTimeElapsed = new AtomicBoolean(false);
private Timer commitTimer;
private boolean fileReadCompletely = true;
@@ -107,6 +85,59 @@ public class HdfsSpout extends BaseRichSpout {
public HdfsSpout() {
}
+ private static String getFileProgress(FileReader reader) {
+ return reader.getFilePath() + " " + reader.getFileOffset();
+ }
+
+ private static void releaseLockAndLog(FileLock fLock, String spoutId) {
+ try {
+ if (fLock != null) {
+ fLock.release();
+ LOG.debug("Spout {} released FileLock. SpoutId = {}", fLock.getLockFile(), spoutId);
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to delete lock file : " + fLock.getLockFile() + " SpoutId =" + spoutId, e);
+ }
+ }
+
+ private static void validateOrMakeDir(FileSystem fs, Path dir, String dirDescription) {
+ try {
+ if (fs.exists(dir)) {
+ if (!fs.isDirectory(dir)) {
+ LOG.error(dirDescription + " directory is a file, not a dir. " + dir);
+ throw new RuntimeException(dirDescription + " directory is a file, not a dir. " + dir);
+ }
+ } else if (!fs.mkdirs(dir)) {
+ LOG.error("Unable to create " + dirDescription + " directory " + dir);
+ throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir);
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to create " + dirDescription + " directory " + dir, e);
+ throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir, e);
+ }
+ }
+
+ static void checkValidReader(String readerType) {
+ if (readerType.equalsIgnoreCase(Configs.TEXT) || readerType.equalsIgnoreCase(Configs.SEQ)) {
+ return;
+ }
+ try {
+ Class<?> classType = Class.forName(readerType);
+ classType.getConstructor(FileSystem.class, Path.class, Map.class);
+ if (!FileReader.class.isAssignableFrom(classType)) {
+ LOG.error(readerType + " not a FileReader");
+ throw new IllegalArgumentException(readerType + " not a FileReader.");
+ }
+ return;
+ } catch (ClassNotFoundException e) {
+ LOG.error(readerType + " not found in classpath.", e);
+ throw new IllegalArgumentException(readerType + " not found in classpath.", e);
+ } catch (NoSuchMethodException e) {
+ LOG.error(readerType + " is missing the expected constructor for Readers.", e);
+ throw new IllegalArgumentException(readerType + " is missing the expected constuctor for Readers.");
+ }
+ }
+
public HdfsSpout setHdfsUri(String hdfsUri) {
this.hdfsUri = hdfsUri;
return this;
@@ -211,8 +242,8 @@ public class HdfsSpout extends BaseRichSpout {
if (ackEnabled && tracker.size() >= maxOutstanding) {
LOG.warn("Waiting for more ACKs before generating new tuples. "
- + "Progress tracker size has reached limit {}, SpoutID {}",
- maxOutstanding, spoutId);
+ + "Progress tracker size has reached limit {}, SpoutID {}",
+ maxOutstanding, spoutId);
// Don't emit anything .. allow configured spout wait strategy to kick in
return;
}
@@ -260,7 +291,7 @@ public class HdfsSpout extends BaseRichSpout {
return;
} catch (ParseException e) {
LOG.error("Parsing error when processing at file location " + getFileProgress(reader)
- + ". Skipping remainder of file.", e);
+ + ". Skipping remainder of file.", e);
markFileAsBad(reader.getFilePath());
// Note: We don't return from this method on ParseException to avoid triggering the
// spout wait strategy (due to no emits). Instead we go back into the loop and
@@ -301,10 +332,6 @@ public class HdfsSpout extends BaseRichSpout {
commitTimer.schedule(timerTask, commitFrequencySec * 1000);
}
- private static String getFileProgress(FileReader reader) {
- return reader.getFilePath() + " " + reader.getFileOffset();
- }
-
private void markFileAsDone(Path filePath) {
try {
Path newFile = renameCompletedFile(reader.getFilePath());
@@ -321,7 +348,8 @@ public class HdfsSpout extends BaseRichSpout {
String originalName = new Path(fileNameMinusSuffix).getName();
Path newFile = new Path(badFilesDirPath + Path.SEPARATOR + originalName);
- LOG.info("Moving bad file {} to {}. Processed it till offset {}. SpoutID= {}", originalName, newFile, tracker.getCommitPosition(), spoutId);
+ LOG.info("Moving bad file {} to {}. Processed it till offset {}. SpoutID= {}", originalName, newFile, tracker.getCommitPosition(),
+ spoutId);
try {
if (!hdfs.rename(file, newFile)) { // seems this can fail by returning false or throwing exception
throw new IOException("Move failed for bad file: " + file); // convert false ret value to exception
@@ -343,17 +371,6 @@ public class HdfsSpout extends BaseRichSpout {
lock = null;
}
- private static void releaseLockAndLog(FileLock fLock, String spoutId) {
- try {
- if (fLock != null) {
- fLock.release();
- LOG.debug("Spout {} released FileLock. SpoutId = {}", fLock.getLockFile(), spoutId);
- }
- } catch (IOException e) {
- LOG.error("Unable to delete lock file : " + fLock.getLockFile() + " SpoutId =" + spoutId, e);
- }
- }
-
protected void emitData(List<Object> tuple, MessageId id) {
LOG.trace("Emitting - {}", id);
@@ -512,48 +529,10 @@ public class HdfsSpout extends BaseRichSpout {
this.commitTimer.cancel();
}
- private static void validateOrMakeDir(FileSystem fs, Path dir, String dirDescription) {
- try {
- if (fs.exists(dir)) {
- if (!fs.isDirectory(dir)) {
- LOG.error(dirDescription + " directory is a file, not a dir. " + dir);
- throw new RuntimeException(dirDescription + " directory is a file, not a dir. " + dir);
- }
- } else if (!fs.mkdirs(dir)) {
- LOG.error("Unable to create " + dirDescription + " directory " + dir);
- throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir);
- }
- } catch (IOException e) {
- LOG.error("Unable to create " + dirDescription + " directory " + dir, e);
- throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir, e);
- }
- }
-
private String getDefaultLockDir(Path sourceDirPath) {
return sourceDirPath.toString() + Path.SEPARATOR + Configs.DEFAULT_LOCK_DIR;
}
- static void checkValidReader(String readerType) {
- if (readerType.equalsIgnoreCase(Configs.TEXT) || readerType.equalsIgnoreCase(Configs.SEQ)) {
- return;
- }
- try {
- Class<?> classType = Class.forName(readerType);
- classType.getConstructor(FileSystem.class, Path.class, Map.class);
- if (!FileReader.class.isAssignableFrom(classType)) {
- LOG.error(readerType + " not a FileReader");
- throw new IllegalArgumentException(readerType + " not a FileReader.");
- }
- return;
- } catch (ClassNotFoundException e) {
- LOG.error(readerType + " not found in classpath.", e);
- throw new IllegalArgumentException(readerType + " not found in classpath.", e);
- } catch (NoSuchMethodException e) {
- LOG.error(readerType + " is missing the expected constructor for Readers.", e);
- throw new IllegalArgumentException(readerType + " is missing the expected constuctor for Readers.");
- }
- }
-
@Override
public void ack(Object msgId) {
LOG.trace("Ack received for msg {} on spout {}", msgId, spoutId);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java
index fdf7751f..a7845ea 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ParseException.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
public class ParseException extends Exception {
- public ParseException(String message, Throwable cause) {
- super(message, cause);
- }
+ public ParseException(String message, Throwable cause) {
+ super(message, cause);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java
index e2e7126..93a9b09 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/ProgressTracker.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
@@ -23,48 +17,48 @@ import java.util.TreeSet;
public class ProgressTracker {
- TreeSet<FileOffset> offsets = new TreeSet<>();
+ TreeSet<FileOffset> offsets = new TreeSet<>();
- public synchronized void recordAckedOffset(FileOffset newOffset) {
- if(newOffset==null) {
- return;
- }
- offsets.add(newOffset);
+ public synchronized void recordAckedOffset(FileOffset newOffset) {
+ if (newOffset == null) {
+ return;
+ }
+ offsets.add(newOffset);
- FileOffset currHead = offsets.first();
+ FileOffset currHead = offsets.first();
- if( currHead.isNextOffset(newOffset) ) { // check is a minor optimization
- trimHead();
+ if (currHead.isNextOffset(newOffset)) { // check is a minor optimization
+ trimHead();
+ }
}
- }
- // remove contiguous elements from the head of the heap
- // e.g.: 1,2,3,4,10,11,12,15 => 4,10,11,12,15
- private synchronized void trimHead() {
- if(offsets.size()<=1) {
- return;
- }
- FileOffset head = offsets.first();
- FileOffset head2 = offsets.higher(head);
- if( head.isNextOffset(head2) ) {
- offsets.pollFirst();
- trimHead();
+ // remove contiguous elements from the head of the heap
+ // e.g.: 1,2,3,4,10,11,12,15 => 4,10,11,12,15
+ private synchronized void trimHead() {
+ if (offsets.size() <= 1) {
+ return;
+ }
+ FileOffset head = offsets.first();
+ FileOffset head2 = offsets.higher(head);
+ if (head.isNextOffset(head2)) {
+ offsets.pollFirst();
+ trimHead();
+ }
+ return;
}
- return;
- }
- public synchronized FileOffset getCommitPosition() {
- if(!offsets.isEmpty()) {
- return offsets.first().clone();
+ public synchronized FileOffset getCommitPosition() {
+ if (!offsets.isEmpty()) {
+ return offsets.first().clone();
+ }
+ return null;
}
- return null;
- }
- public synchronized void dumpState(PrintStream stream) {
- stream.println(offsets);
- }
+ public synchronized void dumpState(PrintStream stream) {
+ stream.println(offsets);
+ }
- public synchronized int size() {
- return offsets.size();
- }
+ public synchronized int size() {
+ return offsets.size();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java
index 64b6b7a..ab61c2b 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/SequenceFileReader.java
@@ -1,23 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
@@ -26,188 +25,185 @@ import org.apache.hadoop.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+public class SequenceFileReader<Key extends Writable, Value extends Writable>
+ extends AbstractFileReader {
+ public static final String[] defaultFields = { "key", "value" };
+ public static final String BUFFER_SIZE = "hdfsspout.reader.buffer.bytes";
+ private static final Logger LOG = LoggerFactory
+ .getLogger(SequenceFileReader.class);
+ private static final int DEFAULT_BUFF_SIZE = 4096;
+ private final SequenceFile.Reader reader;
-public class SequenceFileReader<Key extends Writable,Value extends Writable>
- extends AbstractFileReader {
- private static final Logger LOG = LoggerFactory
- .getLogger(SequenceFileReader.class);
- public static final String[] defaultFields = {"key", "value"};
- private static final int DEFAULT_BUFF_SIZE = 4096;
- public static final String BUFFER_SIZE = "hdfsspout.reader.buffer.bytes";
-
- private final SequenceFile.Reader reader;
-
- private final SequenceFileReader.Offset offset;
-
-
- private final Key key;
- private final Value value;
-
-
- public SequenceFileReader(FileSystem fs, Path file, Map<String, Object> conf)
- throws IOException {
- super(fs, file);
- int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt( conf.get(BUFFER_SIZE).toString() );
- this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize) );
- this.key = (Key) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf() );
- this.value = (Value) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf() );
- this.offset = new SequenceFileReader.Offset(0,0,0);
- }
-
- public SequenceFileReader(FileSystem fs, Path file, Map<String, Object> conf, String offset)
- throws IOException {
- super(fs, file);
- int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt( conf.get(BUFFER_SIZE).toString() );
- this.offset = new SequenceFileReader.Offset(offset);
- this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize) );
- this.key = (Key) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf() );
- this.value = (Value) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf() );
- skipToOffset(this.reader, this.offset, this.key);
- }
-
- private static <K> void skipToOffset(SequenceFile.Reader reader, Offset offset, K key) throws IOException {
- reader.sync(offset.lastSyncPoint);
- for(int i=0; i<offset.recordsSinceLastSync; ++i) {
- reader.next(key);
- }
- }
-
- public List<Object> next() throws IOException, ParseException {
- if( reader.next(key, value) ) {
- ArrayList<Object> result = new ArrayList<Object>(2);
- Collections.addAll(result, key, value);
- offset.increment(reader.syncSeen(), reader.getPosition() );
- return result;
- }
- return null;
- }
-
- @Override
- public void close() {
- try {
- reader.close();
- } catch (IOException e) {
- LOG.warn("Ignoring error when closing file " + getFilePath(), e);
- }
- }
+ private final SequenceFileReader.Offset offset;
- public Offset getFileOffset() {
- return offset;
- }
+ private final Key key;
+ private final Value value;
- public static class Offset implements FileOffset {
- public long lastSyncPoint;
- public long recordsSinceLastSync;
- public long currentRecord;
- private long currRecordEndOffset;
- private long prevRecordEndOffset;
- public Offset(long lastSyncPoint, long recordsSinceLastSync, long currentRecord) {
- this(lastSyncPoint, recordsSinceLastSync, currentRecord, 0, 0 );
+ public SequenceFileReader(FileSystem fs, Path file, Map<String, Object> conf)
+ throws IOException {
+ super(fs, file);
+ int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString());
+ this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize));
+ this.key = (Key) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf());
+ this.value = (Value) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf());
+ this.offset = new SequenceFileReader.Offset(0, 0, 0);
}
- public Offset(long lastSyncPoint, long recordsSinceLastSync, long currentRecord
- , long currRecordEndOffset, long prevRecordEndOffset) {
- this.lastSyncPoint = lastSyncPoint;
- this.recordsSinceLastSync = recordsSinceLastSync;
- this.currentRecord = currentRecord;
- this.prevRecordEndOffset = prevRecordEndOffset;
- this.currRecordEndOffset = currRecordEndOffset;
+ public SequenceFileReader(FileSystem fs, Path file, Map<String, Object> conf, String offset)
+ throws IOException {
+ super(fs, file);
+ int bufferSize = !conf.containsKey(BUFFER_SIZE) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString());
+ this.offset = new SequenceFileReader.Offset(offset);
+ this.reader = new SequenceFile.Reader(fs.getConf(), SequenceFile.Reader.file(file), SequenceFile.Reader.bufferSize(bufferSize));
+ this.key = (Key) ReflectionUtils.newInstance(reader.getKeyClass(), fs.getConf());
+ this.value = (Value) ReflectionUtils.newInstance(reader.getValueClass(), fs.getConf());
+ skipToOffset(this.reader, this.offset, this.key);
}
- public Offset(String offset) {
- try {
- if(offset==null) {
- throw new IllegalArgumentException("offset cannot be null");
- }
- if(offset.equalsIgnoreCase("0")) {
- this.lastSyncPoint = 0;
- this.recordsSinceLastSync = 0;
- this.currentRecord = 0;
- this.prevRecordEndOffset = 0;
- this.currRecordEndOffset = 0;
- } else {
- String[] parts = offset.split(":");
- this.lastSyncPoint = Long.parseLong(parts[0].split("=")[1]);
- this.recordsSinceLastSync = Long.parseLong(parts[1].split("=")[1]);
- this.currentRecord = Long.parseLong(parts[2].split("=")[1]);
- this.prevRecordEndOffset = 0;
- this.currRecordEndOffset = 0;
+ private static <K> void skipToOffset(SequenceFile.Reader reader, Offset offset, K key) throws IOException {
+ reader.sync(offset.lastSyncPoint);
+ for (int i = 0; i < offset.recordsSinceLastSync; ++i) {
+ reader.next(key);
}
- } catch (Exception e) {
- throw new IllegalArgumentException("'" + offset +
- "' cannot be interpreted. It is not in expected format for SequenceFileReader." +
- " Format e.g. {sync=123:afterSync=345:record=67}");
- }
}
- @Override
- public String toString() {
- return '{' +
- "sync=" + lastSyncPoint +
- ":afterSync=" + recordsSinceLastSync +
- ":record=" + currentRecord +
- ":}";
+ public List<Object> next() throws IOException, ParseException {
+ if (reader.next(key, value)) {
+ ArrayList<Object> result = new ArrayList<Object>(2);
+ Collections.addAll(result, key, value);
+ offset.increment(reader.syncSeen(), reader.getPosition());
+ return result;
+ }
+ return null;
}
@Override
- public boolean isNextOffset(FileOffset rhs) {
- if(rhs instanceof Offset) {
- Offset other = ((Offset) rhs);
- return other.currentRecord > currentRecord+1;
- }
- return false;
+ public void close() {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ LOG.warn("Ignoring error when closing file " + getFilePath(), e);
+ }
}
- @Override
- public int compareTo(FileOffset o) {
- Offset rhs = ((Offset) o);
- if(currentRecord<rhs.currentRecord) {
- return -1;
- }
- if(currentRecord==rhs.currentRecord) {
- return 0;
- }
- return 1;
+ public Offset getFileOffset() {
+ return offset;
}
- @Override
- public boolean equals(Object o) {
- if (this == o) { return true; }
- if (!(o instanceof Offset)) { return false; }
- Offset offset = (Offset) o;
+ public static class Offset implements FileOffset {
+ public long lastSyncPoint;
+ public long recordsSinceLastSync;
+ public long currentRecord;
+ private long currRecordEndOffset;
+ private long prevRecordEndOffset;
- return currentRecord == offset.currentRecord;
- }
+ public Offset(long lastSyncPoint, long recordsSinceLastSync, long currentRecord) {
+ this(lastSyncPoint, recordsSinceLastSync, currentRecord, 0, 0);
+ }
- @Override
- public int hashCode() {
- return (int) (currentRecord ^ (currentRecord >>> 32));
- }
-
- void increment(boolean syncSeen, long newBytePosition) {
- if(!syncSeen) {
- ++recordsSinceLastSync;
- } else {
- recordsSinceLastSync = 1;
- lastSyncPoint = prevRecordEndOffset;
- }
- ++currentRecord;
- prevRecordEndOffset = currRecordEndOffset;
- currentRecord = newBytePosition;
- }
+ public Offset(long lastSyncPoint, long recordsSinceLastSync, long currentRecord
+ , long currRecordEndOffset, long prevRecordEndOffset) {
+ this.lastSyncPoint = lastSyncPoint;
+ this.recordsSinceLastSync = recordsSinceLastSync;
+ this.currentRecord = currentRecord;
+ this.prevRecordEndOffset = prevRecordEndOffset;
+ this.currRecordEndOffset = currRecordEndOffset;
+ }
- @Override
- public Offset clone() {
- return new Offset(lastSyncPoint, recordsSinceLastSync, currentRecord, currRecordEndOffset, prevRecordEndOffset);
- }
+ public Offset(String offset) {
+ try {
+ if (offset == null) {
+ throw new IllegalArgumentException("offset cannot be null");
+ }
+ if (offset.equalsIgnoreCase("0")) {
+ this.lastSyncPoint = 0;
+ this.recordsSinceLastSync = 0;
+ this.currentRecord = 0;
+ this.prevRecordEndOffset = 0;
+ this.currRecordEndOffset = 0;
+ } else {
+ String[] parts = offset.split(":");
+ this.lastSyncPoint = Long.parseLong(parts[0].split("=")[1]);
+ this.recordsSinceLastSync = Long.parseLong(parts[1].split("=")[1]);
+ this.currentRecord = Long.parseLong(parts[2].split("=")[1]);
+ this.prevRecordEndOffset = 0;
+ this.currRecordEndOffset = 0;
+ }
+ } catch (Exception e) {
+ throw new IllegalArgumentException("'" + offset +
+ "' cannot be interpreted. It is not in expected format for SequenceFileReader." +
+ " Format e.g. {sync=123:afterSync=345:record=67}");
+ }
+ }
+
+ @Override
+ public String toString() {
+ return '{' +
+ "sync=" + lastSyncPoint +
+ ":afterSync=" + recordsSinceLastSync +
+ ":record=" + currentRecord +
+ ":}";
+ }
+
+ @Override
+ public boolean isNextOffset(FileOffset rhs) {
+ if (rhs instanceof Offset) {
+ Offset other = ((Offset) rhs);
+ return other.currentRecord > currentRecord + 1;
+ }
+ return false;
+ }
+
+ @Override
+ public int compareTo(FileOffset o) {
+ Offset rhs = ((Offset) o);
+ if (currentRecord < rhs.currentRecord) {
+ return -1;
+ }
+ if (currentRecord == rhs.currentRecord) {
+ return 0;
+ }
+ return 1;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof Offset)) {
+ return false;
+ }
+
+ Offset offset = (Offset) o;
+
+ return currentRecord == offset.currentRecord;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) (currentRecord ^ (currentRecord >>> 32));
+ }
+
+ void increment(boolean syncSeen, long newBytePosition) {
+ if (!syncSeen) {
+ ++recordsSinceLastSync;
+ } else {
+ recordsSinceLastSync = 1;
+ lastSyncPoint = prevRecordEndOffset;
+ }
+ ++currentRecord;
+ prevRecordEndOffset = currRecordEndOffset;
+ currentRecord = newBytePosition;
+ }
+
+ @Override
+ public Offset clone() {
+ return new Offset(lastSyncPoint, recordsSinceLastSync, currentRecord, currRecordEndOffset, prevRecordEndOffset);
+ }
- } //class Offset
+ } //class Offset
} //class
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java
index a393238..cc5531e 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/TextFileReader.java
@@ -1,192 +1,190 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
// Todo: Track file offsets instead of line number
public class TextFileReader extends AbstractFileReader {
- public static final String[] defaultFields = {"line"};
- public static final String CHARSET = "hdfsspout.reader.charset";
- public static final String BUFFER_SIZE = "hdfsspout.reader.buffer.bytes";
-
- private static final int DEFAULT_BUFF_SIZE = 4096;
-
- private BufferedReader reader;
- private final Logger LOG = LoggerFactory.getLogger(TextFileReader.class);
- private TextFileReader.Offset offset;
-
- public TextFileReader(FileSystem fs, Path file, Map<String, Object> conf) throws IOException {
- this(fs, file, conf, new TextFileReader.Offset(0,0) );
- }
-
- public TextFileReader(FileSystem fs, Path file, Map<String, Object> conf, String startOffset) throws IOException {
- this(fs, file, conf, new TextFileReader.Offset(startOffset) );
- }
-
- private TextFileReader(FileSystem fs, Path file, Map<String, Object> conf, TextFileReader.Offset startOffset)
- throws IOException {
- super(fs, file);
- offset = startOffset;
- FSDataInputStream in = fs.open(file);
-
- String charSet = (conf==null || !conf.containsKey(CHARSET) ) ? "UTF-8" : conf.get(CHARSET).toString();
- int buffSz = (conf==null || !conf.containsKey(BUFFER_SIZE) ) ? DEFAULT_BUFF_SIZE : Integer.parseInt( conf.get(BUFFER_SIZE).toString() );
- reader = new BufferedReader(new InputStreamReader(in, charSet), buffSz);
- if(offset.charOffset >0) {
- reader.skip(offset.charOffset);
- }
-
- }
+ public static final String[] defaultFields = { "line" };
+ public static final String CHARSET = "hdfsspout.reader.charset";
+ public static final String BUFFER_SIZE = "hdfsspout.reader.buffer.bytes";
- public Offset getFileOffset() {
- return offset.clone();
- }
+ private static final int DEFAULT_BUFF_SIZE = 4096;
+ private final Logger LOG = LoggerFactory.getLogger(TextFileReader.class);
+ private BufferedReader reader;
+ private TextFileReader.Offset offset;
- public List<Object> next() throws IOException, ParseException {
- String line = readLineAndTrackOffset(reader);
- if(line!=null) {
- return Collections.singletonList((Object) line);
- }
- return null;
- }
-
- private String readLineAndTrackOffset(BufferedReader reader) throws IOException {
- StringBuffer sb = new StringBuffer(1000);
- long before = offset.charOffset;
- int ch;
- while( (ch = reader.read()) != -1 ) {
- ++offset.charOffset;
- if (ch == '\n') {
- ++offset.lineNumber;
- return sb.toString();
- } else if( ch != '\r') {
- sb.append((char)ch);
- }
- }
- if(before==offset.charOffset) { // reached EOF, didnt read anything
- return null;
+ public TextFileReader(FileSystem fs, Path file, Map<String, Object> conf) throws IOException {
+ this(fs, file, conf, new TextFileReader.Offset(0, 0));
}
- return sb.toString();
- }
-
- @Override
- public void close() {
- try {
- reader.close();
- } catch (IOException e) {
- LOG.warn("Ignoring error when closing file " + getFilePath(), e);
+
+ public TextFileReader(FileSystem fs, Path file, Map<String, Object> conf, String startOffset) throws IOException {
+ this(fs, file, conf, new TextFileReader.Offset(startOffset));
}
- }
- public static class Offset implements FileOffset {
- long charOffset;
- long lineNumber;
+ private TextFileReader(FileSystem fs, Path file, Map<String, Object> conf, TextFileReader.Offset startOffset)
+ throws IOException {
+ super(fs, file);
+ offset = startOffset;
+ FSDataInputStream in = fs.open(file);
+
+ String charSet = (conf == null || !conf.containsKey(CHARSET)) ? "UTF-8" : conf.get(CHARSET).toString();
+ int buffSz =
+ (conf == null || !conf.containsKey(BUFFER_SIZE)) ? DEFAULT_BUFF_SIZE : Integer.parseInt(conf.get(BUFFER_SIZE).toString());
+ reader = new BufferedReader(new InputStreamReader(in, charSet), buffSz);
+ if (offset.charOffset > 0) {
+ reader.skip(offset.charOffset);
+ }
- public Offset(long byteOffset, long lineNumber) {
- this.charOffset = byteOffset;
- this.lineNumber = lineNumber;
}
- public Offset(String offset) {
- if(offset==null) {
- throw new IllegalArgumentException("offset cannot be null");
- }
- try {
- if(offset.equalsIgnoreCase("0")) {
- this.charOffset = 0;
- this.lineNumber = 0;
- } else {
- String[] parts = offset.split(":");
- this.charOffset = Long.parseLong(parts[0].split("=")[1]);
- this.lineNumber = Long.parseLong(parts[1].split("=")[1]);
- }
- } catch (Exception e) {
- throw new IllegalArgumentException("'" + offset +
- "' cannot be interpreted. It is not in expected format for TextFileReader." +
- " Format e.g. {char=123:line=5}");
- }
+ public Offset getFileOffset() {
+ return offset.clone();
}
- @Override
- public String toString() {
- return '{' +
- "char=" + charOffset +
- ":line=" + lineNumber +
- ":}";
+ public List<Object> next() throws IOException, ParseException {
+ String line = readLineAndTrackOffset(reader);
+ if (line != null) {
+ return Collections.singletonList((Object) line);
+ }
+ return null;
}
- @Override
- public boolean isNextOffset(FileOffset rhs) {
- if(rhs instanceof Offset) {
- Offset other = ((Offset) rhs);
- return other.charOffset > charOffset &&
- other.lineNumber == lineNumber+1;
- }
- return false;
+ private String readLineAndTrackOffset(BufferedReader reader) throws IOException {
+ StringBuffer sb = new StringBuffer(1000);
+ long before = offset.charOffset;
+ int ch;
+ while ((ch = reader.read()) != -1) {
+ ++offset.charOffset;
+ if (ch == '\n') {
+ ++offset.lineNumber;
+ return sb.toString();
+ } else if (ch != '\r') {
+ sb.append((char) ch);
+ }
+ }
+ if (before == offset.charOffset) { // reached EOF, didnt read anything
+ return null;
+ }
+ return sb.toString();
}
@Override
- public int compareTo(FileOffset o) {
- Offset rhs = ((Offset)o);
- if(lineNumber < rhs.lineNumber) {
- return -1;
- }
- if(lineNumber == rhs.lineNumber) {
- return 0;
- }
- return 1;
+ public void close() {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ LOG.warn("Ignoring error when closing file " + getFilePath(), e);
+ }
}
- @Override
- public boolean equals(Object o) {
- if (this == o) { return true; }
- if (!(o instanceof Offset)) { return false; }
+ public static class Offset implements FileOffset {
+ long charOffset;
+ long lineNumber;
- Offset that = (Offset) o;
+ public Offset(long byteOffset, long lineNumber) {
+ this.charOffset = byteOffset;
+ this.lineNumber = lineNumber;
+ }
- if (charOffset != that.charOffset)
- return false;
- return lineNumber == that.lineNumber;
- }
+ public Offset(String offset) {
+ if (offset == null) {
+ throw new IllegalArgumentException("offset cannot be null");
+ }
+ try {
+ if (offset.equalsIgnoreCase("0")) {
+ this.charOffset = 0;
+ this.lineNumber = 0;
+ } else {
+ String[] parts = offset.split(":");
+ this.charOffset = Long.parseLong(parts[0].split("=")[1]);
+ this.lineNumber = Long.parseLong(parts[1].split("=")[1]);
+ }
+ } catch (Exception e) {
+ throw new IllegalArgumentException("'" + offset +
+ "' cannot be interpreted. It is not in expected format for TextFileReader." +
+ " Format e.g. {char=123:line=5}");
+ }
+ }
- @Override
- public int hashCode() {
- int result = (int) (charOffset ^ (charOffset >>> 32));
- result = 31 * result + (int) (lineNumber ^ (lineNumber >>> 32));
- return result;
- }
+ @Override
+ public String toString() {
+ return '{' +
+ "char=" + charOffset +
+ ":line=" + lineNumber +
+ ":}";
+ }
- @Override
- public Offset clone() {
- return new Offset(charOffset, lineNumber);
- }
- } //class Offset
+ @Override
+ public boolean isNextOffset(FileOffset rhs) {
+ if (rhs instanceof Offset) {
+ Offset other = ((Offset) rhs);
+ return other.charOffset > charOffset &&
+ other.lineNumber == lineNumber + 1;
+ }
+ return false;
+ }
+
+ @Override
+ public int compareTo(FileOffset o) {
+ Offset rhs = ((Offset) o);
+ if (lineNumber < rhs.lineNumber) {
+ return -1;
+ }
+ if (lineNumber == rhs.lineNumber) {
+ return 0;
+ }
+ return 1;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof Offset)) {
+ return false;
+ }
+
+ Offset that = (Offset) o;
+
+ if (charOffset != that.charOffset) {
+ return false;
+ }
+ return lineNumber == that.lineNumber;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = (int) (charOffset ^ (charOffset >>> 32));
+ result = 31 * result + (int) (lineNumber ^ (lineNumber >>> 32));
+ return result;
+ }
+
+ @Override
+ public Offset clone() {
+ return new Offset(charOffset, lineNumber);
+ }
+ } //class Offset
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
index 07968f2..e6adfbb 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
@@ -15,8 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.trident;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -43,32 +55,154 @@ import org.apache.storm.trident.tuple.TridentTuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.Serializable;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-
public class HdfsState implements State {
+ public static final Logger LOG = LoggerFactory.getLogger(HdfsState.class);
+ private Options options;
+ private volatile TxnRecord lastSeenTxn;
+ private Path indexFilePath;
+
+
+ HdfsState(Options options) {
+ this.options = options;
+ }
+
+ void prepare(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ this.options.prepare(conf, partitionIndex, numPartitions);
+ initLastTxn(conf, partitionIndex);
+ }
+
+ private TxnRecord readTxnRecord(Path path) throws IOException {
+ FSDataInputStream inputStream = null;
+ try {
+ inputStream = this.options.fs.open(path);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+ String line;
+ if ((line = reader.readLine()) != null) {
+ String[] fields = line.split(",");
+ return new TxnRecord(Long.valueOf(fields[0]), fields[1], Long.valueOf(fields[2]));
+ }
+ } finally {
+ if (inputStream != null) {
+ inputStream.close();
+ }
+ }
+ return new TxnRecord(0, options.currentFile.toString(), 0);
+ }
+
+ /**
+ * Returns temp file path corresponding to a file name.
+ */
+ private Path tmpFilePath(String filename) {
+ return new Path(filename + ".tmp");
+ }
+
+ /**
+ * Reads the last txn record from index file if it exists, if not from .tmp file if exists.
+ *
+ * @param indexFilePath the index file path
+ * @return the txn record from the index file or a default initial record.
+ *
+ * @throws IOException
+ */
+ private TxnRecord getTxnRecord(Path indexFilePath) throws IOException {
+ Path tmpPath = tmpFilePath(indexFilePath.toString());
+ if (this.options.fs.exists(indexFilePath)) {
+ return readTxnRecord(indexFilePath);
+ } else if (this.options.fs.exists(tmpPath)) {
+ return readTxnRecord(tmpPath);
+ }
+ return new TxnRecord(0, options.currentFile.toString(), 0);
+ }
+
+ private void initLastTxn(Map<String, Object> conf, int partition) {
+ // include partition id in the file name so that index for different partitions are independent.
+ String indexFileName = String.format(".index.%s.%d", conf.get(Config.TOPOLOGY_NAME), partition);
+ this.indexFilePath = new Path(options.fileNameFormat.getPath(), indexFileName);
+ try {
+ this.lastSeenTxn = getTxnRecord(indexFilePath);
+ LOG.debug("initLastTxn updated lastSeenTxn to [{}]", this.lastSeenTxn);
+ } catch (IOException e) {
+ LOG.warn("initLastTxn failed due to IOException.", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void updateIndex(long txId) {
+ LOG.debug("Starting index update.");
+ final Path tmpPath = tmpFilePath(indexFilePath.toString());
+
+ try (FSDataOutputStream out = this.options.fs.create(tmpPath, true);
+ BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out))) {
+ TxnRecord txnRecord = new TxnRecord(txId, options.currentFile.toString(), this.options.getCurrentOffset());
+ bw.write(txnRecord.toString());
+ bw.newLine();
+ bw.flush();
+ out.close(); /* In non error scenarios, for the Azure Data Lake Store File System (adl://),
+ the output stream must be closed before the file associated with it is deleted.
+ For ADLFS deleting the file also removes any handles to the file, hence out.close() will fail. */
+ /*
+ * Delete the current index file and rename the tmp file to atomically
+ * replace the index file. Orphan .tmp files are handled in getTxnRecord.
+ */
+ options.fs.delete(this.indexFilePath, false);
+ options.fs.rename(tmpPath, this.indexFilePath);
+ lastSeenTxn = txnRecord;
+ LOG.debug("updateIndex updated lastSeenTxn to [{}]", this.lastSeenTxn);
+ } catch (IOException e) {
+ LOG.warn("Begin commit failed due to IOException. Failing batch", e);
+ throw new FailedException(e);
+ }
+ }
+
+ @Override
+ public void beginCommit(Long txId) {
+ if (txId <= lastSeenTxn.txnid) {
+ LOG.info("txID {} is already processed, lastSeenTxn {}. Triggering recovery.", txId, lastSeenTxn);
+ long start = System.currentTimeMillis();
+ options.recover(lastSeenTxn.dataFilePath, lastSeenTxn.offset);
+ LOG.info("Recovery took {} ms.", System.currentTimeMillis() - start);
+ }
+ updateIndex(txId);
+ }
+
+ @Override
+ public void commit(Long txId) {
+ try {
+ options.doCommit(txId);
+ } catch (IOException e) {
+ LOG.warn("Commit failed due to IOException. Failing the batch.", e);
+ throw new FailedException(e);
+ }
+ }
+
+ public void updateState(List<TridentTuple> tuples, TridentCollector tridentCollector) {
+ try {
+ this.options.execute(tuples);
+ } catch (IOException e) {
+ LOG.warn("Failing batch due to IOException.", e);
+ throw new FailedException(e);
+ }
+ }
+
+ /**
+ * for unit tests
+ */
+ void close() throws IOException {
+ this.options.closeOutputFile();
+ }
+
public static abstract class Options implements Serializable {
protected String fsUrl;
protected String configKey;
protected transient FileSystem fs;
- private Path currentFile;
protected FileRotationPolicy rotationPolicy;
protected FileNameFormat fileNameFormat;
protected int rotation = 0;
protected transient Configuration hdfsConfig;
protected ArrayList<RotationAction> rotationActions = new ArrayList<RotationAction>();
-
+ private Path currentFile;
abstract void closeOutputFile() throws IOException;
@@ -78,7 +212,7 @@ public class HdfsState implements State {
abstract void doPrepare(Map<String, Object> conf, int partitionIndex, int numPartitions) throws IOException;
- abstract long getCurrentOffset() throws IOException;
+ abstract long getCurrentOffset() throws IOException;
abstract void doCommit(Long txId) throws IOException;
@@ -143,8 +277,7 @@ public class HdfsState implements State {
}
/**
- * Recovers nBytes from srcFile to the new file created
- * by calling rotateOutputFile and then deletes the srcFile.
+ * Recovers nBytes from srcFile to the new file created by calling rotateOutputFile and then deletes the srcFile.
*/
private void recover(String srcFile, long nBytes) {
try {
@@ -169,10 +302,10 @@ public class HdfsState implements State {
public static class HdfsFileOptions extends Options {
- private transient FSDataOutputStream out;
protected RecordFormat format;
+ private transient FSDataOutputStream out;
private long offset = 0;
- private int bufferSize = 131072; // default 128 K
+ private int bufferSize = 131072; // default 128 K
public HdfsFileOptions withFsUrl(String fsUrl) {
this.fsUrl = fsUrl;
@@ -360,24 +493,25 @@ public class HdfsState implements State {
@Override
void doRecover(Path srcPath, long nBytes) throws Exception {
SequenceFile.Reader reader = new SequenceFile.Reader(this.hdfsConfig,
- SequenceFile.Reader.file(srcPath), SequenceFile.Reader.length(nBytes));
+ SequenceFile.Reader.file(srcPath), SequenceFile.Reader.length(nBytes));
Writable key = (Writable) this.format.keyClass().newInstance();
Writable value = (Writable) this.format.valueClass().newInstance();
- while(reader.next(key, value)) {
+ while (reader.next(key, value)) {
this.writer.append(key, value);
}
}
@Override
Path createOutputFile() throws IOException {
- Path p = new Path(this.fsUrl + this.fileNameFormat.getPath(), this.fileNameFormat.getName(this.rotation, System.currentTimeMillis()));
+ Path p = new Path(this.fsUrl + this.fileNameFormat.getPath(),
+ this.fileNameFormat.getName(this.rotation, System.currentTimeMillis()));
this.writer = SequenceFile.createWriter(
- this.hdfsConfig,
- SequenceFile.Writer.file(p),
- SequenceFile.Writer.keyClass(this.format.keyClass()),
- SequenceFile.Writer.valueClass(this.format.valueClass()),
- SequenceFile.Writer.compression(this.compressionType, this.codecFactory.getCodecByName(this.compressionCodec))
+ this.hdfsConfig,
+ SequenceFile.Writer.file(p),
+ SequenceFile.Writer.keyClass(this.format.keyClass()),
+ SequenceFile.Writer.valueClass(this.format.valueClass()),
+ SequenceFile.Writer.compression(this.compressionType, this.codecFactory.getCodecByName(this.compressionCodec))
);
return p;
}
@@ -418,138 +552,4 @@ public class HdfsState implements State {
return Long.toString(txnid) + "," + dataFilePath + "," + Long.toString(offset);
}
}
-
-
- public static final Logger LOG = LoggerFactory.getLogger(HdfsState.class);
- private Options options;
- private volatile TxnRecord lastSeenTxn;
- private Path indexFilePath;
-
- HdfsState(Options options) {
- this.options = options;
- }
-
- void prepare(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
- this.options.prepare(conf, partitionIndex, numPartitions);
- initLastTxn(conf, partitionIndex);
- }
-
- private TxnRecord readTxnRecord(Path path) throws IOException {
- FSDataInputStream inputStream = null;
- try {
- inputStream = this.options.fs.open(path);
- BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
- String line;
- if ((line = reader.readLine()) != null) {
- String[] fields = line.split(",");
- return new TxnRecord(Long.valueOf(fields[0]), fields[1], Long.valueOf(fields[2]));
- }
- } finally {
- if (inputStream != null) {
- inputStream.close();
- }
- }
- return new TxnRecord(0, options.currentFile.toString(), 0);
- }
-
- /**
- * Returns temp file path corresponding to a file name.
- */
- private Path tmpFilePath(String filename) {
- return new Path(filename + ".tmp");
- }
- /**
- * Reads the last txn record from index file if it exists, if not
- * from .tmp file if exists.
- *
- * @param indexFilePath the index file path
- * @return the txn record from the index file or a default initial record.
- * @throws IOException
- */
- private TxnRecord getTxnRecord(Path indexFilePath) throws IOException {
- Path tmpPath = tmpFilePath(indexFilePath.toString());
- if (this.options.fs.exists(indexFilePath)) {
- return readTxnRecord(indexFilePath);
- } else if (this.options.fs.exists(tmpPath)) {
- return readTxnRecord(tmpPath);
- }
- return new TxnRecord(0, options.currentFile.toString(), 0);
- }
-
- private void initLastTxn(Map<String, Object> conf, int partition) {
- // include partition id in the file name so that index for different partitions are independent.
- String indexFileName = String.format(".index.%s.%d", conf.get(Config.TOPOLOGY_NAME), partition);
- this.indexFilePath = new Path(options.fileNameFormat.getPath(), indexFileName);
- try {
- this.lastSeenTxn = getTxnRecord(indexFilePath);
- LOG.debug("initLastTxn updated lastSeenTxn to [{}]", this.lastSeenTxn);
- } catch (IOException e) {
- LOG.warn("initLastTxn failed due to IOException.", e);
- throw new RuntimeException(e);
- }
- }
-
- private void updateIndex(long txId) {
- LOG.debug("Starting index update.");
- final Path tmpPath = tmpFilePath(indexFilePath.toString());
-
- try (FSDataOutputStream out = this.options.fs.create(tmpPath, true);
- BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out))) {
- TxnRecord txnRecord = new TxnRecord(txId, options.currentFile.toString(), this.options.getCurrentOffset());
- bw.write(txnRecord.toString());
- bw.newLine();
- bw.flush();
- out.close(); /* In non error scenarios, for the Azure Data Lake Store File System (adl://),
- the output stream must be closed before the file associated with it is deleted.
- For ADLFS deleting the file also removes any handles to the file, hence out.close() will fail. */
- /*
- * Delete the current index file and rename the tmp file to atomically
- * replace the index file. Orphan .tmp files are handled in getTxnRecord.
- */
- options.fs.delete(this.indexFilePath, false);
- options.fs.rename(tmpPath, this.indexFilePath);
- lastSeenTxn = txnRecord;
- LOG.debug("updateIndex updated lastSeenTxn to [{}]", this.lastSeenTxn);
- } catch (IOException e) {
- LOG.warn("Begin commit failed due to IOException. Failing batch", e);
- throw new FailedException(e);
- }
- }
-
- @Override
- public void beginCommit(Long txId) {
- if (txId <= lastSeenTxn.txnid) {
- LOG.info("txID {} is already processed, lastSeenTxn {}. Triggering recovery.", txId, lastSeenTxn);
- long start = System.currentTimeMillis();
- options.recover(lastSeenTxn.dataFilePath, lastSeenTxn.offset);
- LOG.info("Recovery took {} ms.", System.currentTimeMillis() - start);
- }
- updateIndex(txId);
- }
-
- @Override
- public void commit(Long txId) {
- try {
- options.doCommit(txId);
- } catch (IOException e) {
- LOG.warn("Commit failed due to IOException. Failing the batch.", e);
- throw new FailedException(e);
- }
- }
-
- public void updateState(List<TridentTuple> tuples, TridentCollector tridentCollector) {
- try {
- this.options.execute(tuples);
- } catch (IOException e) {
- LOG.warn("Failing batch due to IOException.", e);
- throw new FailedException(e);
- }
- }
-
- /**
- * for unit tests
- */
- void close() throws IOException {
- this.options.closeOutputFile();
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java
index e76ec22..568f8bc 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsStateFactory.java
@@ -15,23 +15,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.trident;
+import java.util.Map;
import org.apache.storm.task.IMetricsContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.state.StateFactory;
-
-import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HdfsStateFactory implements StateFactory {
private static final Logger LOG = LoggerFactory.getLogger(HdfsStateFactory.class);
private HdfsState.Options options;
- public HdfsStateFactory(){}
+ public HdfsStateFactory() {}
- public HdfsStateFactory withOptions(HdfsState.Options options){
+ public HdfsStateFactory withOptions(HdfsState.Options options) {
this.options = options;
return this;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java
index c603334..a63bb40 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/HdfsUpdater.java
@@ -15,15 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.trident;
+import java.util.List;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseStateUpdater;
import org.apache.storm.trident.tuple.TridentTuple;
-import java.util.List;
-
-public class HdfsUpdater extends BaseStateUpdater<HdfsState>{
+public class HdfsUpdater extends BaseStateUpdater<HdfsState> {
@Override
public void updateState(HdfsState state, List<TridentTuple> tuples, TridentCollector collector) {
state.updateState(tuples, collector);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java
index a952b36..825a0f0 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultFileNameFormat.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.trident.format;
import java.util.Map;
@@ -45,7 +40,7 @@ public class DefaultFileNameFormat implements FileNameFormat {
* @param prefix
* @return
*/
- public DefaultFileNameFormat withPrefix(String prefix){
+ public DefaultFileNameFormat withPrefix(String prefix) {
this.prefix = prefix;
return this;
}
@@ -56,12 +51,12 @@ public class DefaultFileNameFormat implements FileNameFormat {
* @param extension
* @return
*/
- public DefaultFileNameFormat withExtension(String extension){
+ public DefaultFileNameFormat withExtension(String extension) {
this.extension = extension;
return this;
}
- public DefaultFileNameFormat withPath(String path){
+ public DefaultFileNameFormat withPath(String path) {
this.path = path;
return this;
}
@@ -74,10 +69,10 @@ public class DefaultFileNameFormat implements FileNameFormat {
@Override
public String getName(long rotation, long timeStamp) {
- return this.prefix + "-" + this.partitionIndex + "-" + rotation + "-" + timeStamp + this.extension;
+ return this.prefix + "-" + this.partitionIndex + "-" + rotation + "-" + timeStamp + this.extension;
}
- public String getPath(){
+ public String getPath() {
return this.path;
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java
index 1336144..f33c030 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DefaultSequenceFormat.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.trident.format;
import org.apache.hadoop.io.LongWritable;
@@ -25,7 +26,6 @@ import org.apache.storm.trident.tuple.TridentTuple;
/**
* Basic <code>SequenceFormat</code> implementation that uses
* <code>LongWritable</code> for keys and <code>Text</code> for values.
- *
*/
public class DefaultSequenceFormat implements SequenceFormat {
private transient LongWritable key;
@@ -34,13 +34,12 @@ public class DefaultSequenceFormat implements SequenceFormat {
private String keyField;
private String valueField;
- public DefaultSequenceFormat(String keyField, String valueField){
+ public DefaultSequenceFormat(String keyField, String valueField) {
this.keyField = keyField;
this.valueField = valueField;
}
-
@Override
public Class keyClass() {
return LongWritable.class;
@@ -53,8 +52,8 @@ public class DefaultSequenceFormat implements SequenceFormat {
@Override
public Writable key(TridentTuple tuple) {
- if(this.key == null){
- this.key = new LongWritable();
+ if (this.key == null) {
+ this.key = new LongWritable();
}
this.key.set(tuple.getLongByField(this.keyField));
return this.key;
@@ -62,7 +61,7 @@ public class DefaultSequenceFormat implements SequenceFormat {
@Override
public Writable value(TridentTuple tuple) {
- if(this.value == null){
+ if (this.value == null) {
this.value = new Text();
}
this.value.set(tuple.getStringByField(this.valueField));
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java
index a08664d..e21fede 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/DelimitedRecordFormat.java
@@ -1,24 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.trident.format;
-import org.apache.storm.tuple.Fields;
import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Fields;
/**
* RecordFormat implementation that uses field and record delimiters.
@@ -40,7 +35,7 @@ public class DelimitedRecordFormat implements RecordFormat {
* @param fields
* @return
*/
- public DelimitedRecordFormat withFields(Fields fields){
+ public DelimitedRecordFormat withFields(Fields fields) {
this.fields = fields;
return this;
}
@@ -51,7 +46,7 @@ public class DelimitedRecordFormat implements RecordFormat {
* @param delimiter
* @return
*/
- public DelimitedRecordFormat withFieldDelimiter(String delimiter){
+ public DelimitedRecordFormat withFieldDelimiter(String delimiter) {
this.fieldDelimiter = delimiter;
return this;
}
@@ -62,7 +57,7 @@ public class DelimitedRecordFormat implements RecordFormat {
* @param delimiter
* @return
*/
- public DelimitedRecordFormat withRecordDelimiter(String delimiter){
+ public DelimitedRecordFormat withRecordDelimiter(String delimiter) {
this.recordDelimiter = delimiter;
return this;
}
@@ -71,9 +66,9 @@ public class DelimitedRecordFormat implements RecordFormat {
public byte[] format(TridentTuple tuple) {
StringBuilder sb = new StringBuilder();
int size = this.fields.size();
- for(int i = 0; i < size; i++){
+ for (int i = 0; i < size; i++) {
sb.append(tuple.getValueByField(fields.get(i)));
- if(i != size - 1){
+ if (i != size - 1) {
sb.append(this.fieldDelimiter);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/FileNameFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/FileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/FileNameFormat.java
index c5b0698..fbd8f5a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/FileNameFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/FileNameFormat.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.trident.format;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/RecordFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/RecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/RecordFormat.java
index 76179d9..1cc5363 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/RecordFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/RecordFormat.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.trident.format;
+package org.apache.storm.hdfs.trident.format;
-import org.apache.storm.trident.tuple.TridentTuple;
import java.io.Serializable;
+import org.apache.storm.trident.tuple.TridentTuple;
/**
* Formats a Tuple object into a byte array
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SequenceFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SequenceFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SequenceFormat.java
index b4d6c5c..497d045 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SequenceFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SequenceFormat.java
@@ -15,16 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.trident.format;
+import java.io.Serializable;
import org.apache.hadoop.io.Writable;
import org.apache.storm.trident.tuple.TridentTuple;
-import java.io.Serializable;
-
/**
* Interface for converting <code>TridentTuple</code> objects to HDFS sequence file key-value pairs.
- *
*/
public interface SequenceFormat extends Serializable {
/**
@@ -36,6 +35,7 @@ public interface SequenceFormat extends Serializable {
/**
* Value class used by implementation (e.g. Text.class, etc.)
+ *
* @return
*/
Class valueClass();
@@ -50,6 +50,7 @@ public interface SequenceFormat extends Serializable {
/**
* Given a tuple, return the value that should be written to the sequence file.
+ *
* @param tuple
* @return
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SimpleFileNameFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SimpleFileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SimpleFileNameFormat.java
index c676324..068390f 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SimpleFileNameFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/trident/format/SimpleFileNameFormat.java
@@ -1,27 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.trident.format;
import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Map;
-
import org.apache.storm.utils.Utils;
public class SimpleFileNameFormat implements FileNameFormat {
@@ -39,10 +33,10 @@ public class SimpleFileNameFormat implements FileNameFormat {
// compile parameters
SimpleDateFormat dateFormat = new SimpleDateFormat(timeFormat);
String ret = name
- .replace("$TIME", dateFormat.format(new Date(timeStamp)))
- .replace("$NUM", String.valueOf(rotation))
- .replace("$HOST", host)
- .replace("$PARTITION", String.valueOf(partitionIndex));
+ .replace("$TIME", dateFormat.format(new Date(timeStamp)))
+ .replace("$NUM", String.valueOf(rotation))
+ .replace("$HOST", host)
+ .replace("$PARTITION", String.valueOf(partitionIndex));
return ret;
}
@@ -73,7 +67,7 @@ public class SimpleFileNameFormat implements FileNameFormat {
* $NUM - rotation number<br/>
* $HOST - local host name<br/>
* $PARTITION - partition index<br/>
- *
+ *
* @param name
* file name
* @return
@@ -85,10 +79,10 @@ public class SimpleFileNameFormat implements FileNameFormat {
public SimpleFileNameFormat withTimeFormat(String timeFormat) {
//check format
- try{
+ try {
new SimpleDateFormat(timeFormat);
- }catch (Exception e) {
- throw new IllegalArgumentException("invalid timeFormat: "+e.getMessage());
+ } catch (Exception e) {
+ throw new IllegalArgumentException("invalid timeFormat: " + e.getMessage());
}
this.timeFormat = timeFormat;
return this;
[10/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/pacemaker/codec/ThriftNettyServerCodec.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/pacemaker/codec/ThriftNettyServerCodec.java b/storm-server/src/main/java/org/apache/storm/pacemaker/codec/ThriftNettyServerCodec.java
index a6ceade..3fa40e6 100644
--- a/storm-server/src/main/java/org/apache/storm/pacemaker/codec/ThriftNettyServerCodec.java
+++ b/storm-server/src/main/java/org/apache/storm/pacemaker/codec/ThriftNettyServerCodec.java
@@ -1,22 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.pacemaker.codec;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
import org.apache.storm.DaemonConfig;
import org.apache.storm.messaging.netty.ISaslServer;
import org.apache.storm.messaging.netty.IServer;
@@ -24,9 +22,6 @@ import org.apache.storm.messaging.netty.KerberosSaslServerHandler;
import org.apache.storm.messaging.netty.SaslStormServerHandler;
import org.apache.storm.messaging.netty.StormServerHandler;
import org.apache.storm.security.auth.AuthUtils;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Map;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
@@ -37,20 +32,14 @@ public class ThriftNettyServerCodec {
public static final String SASL_HANDLER = "sasl-handler";
public static final String KERBEROS_HANDLER = "kerberos-handler";
-
- public enum AuthMethod {
- DIGEST,
- KERBEROS,
- NONE
- };
-
+ private static final Logger LOG = LoggerFactory
+ .getLogger(ThriftNettyServerCodec.class);
+
+ ;
+ private final int thriftMessageMaxSize;
private IServer server;
private AuthMethod authMethod;
private Map<String, Object> topoConf;
- private final int thriftMessageMaxSize;
-
- private static final Logger LOG = LoggerFactory
- .getLogger(ThriftNettyServerCodec.class);
public ThriftNettyServerCodec(IServer server, Map<String, Object> topoConf,
AuthMethod authMethod, int thriftMessageMaxSizeBytes) {
@@ -67,30 +56,26 @@ public class ThriftNettyServerCodec {
ChannelPipeline pipeline = Channels.pipeline();
pipeline.addLast("encoder", new ThriftEncoder());
pipeline.addLast("decoder", new ThriftDecoder(thriftMessageMaxSize));
- if(authMethod == AuthMethod.DIGEST) {
+ if (authMethod == AuthMethod.DIGEST) {
try {
LOG.debug("Adding SaslStormServerHandler to pacemaker server pipeline.");
- pipeline.addLast(SASL_HANDLER, new SaslStormServerHandler((ISaslServer)server));
- }
- catch (IOException e) {
+ pipeline.addLast(SASL_HANDLER, new SaslStormServerHandler((ISaslServer) server));
+ } catch (IOException e) {
throw new RuntimeException(e);
}
- }
- else if(authMethod == AuthMethod.KERBEROS) {
+ } else if (authMethod == AuthMethod.KERBEROS) {
try {
LOG.debug("Adding KerberosSaslServerHandler to pacemaker server pipeline.");
ArrayList<String> authorizedUsers = new ArrayList(1);
- authorizedUsers.add((String)topoConf.get(DaemonConfig.NIMBUS_DAEMON_USER));
- pipeline.addLast(KERBEROS_HANDLER, new KerberosSaslServerHandler((ISaslServer)server,
+ authorizedUsers.add((String) topoConf.get(DaemonConfig.NIMBUS_DAEMON_USER));
+ pipeline.addLast(KERBEROS_HANDLER, new KerberosSaslServerHandler((ISaslServer) server,
topoConf,
AuthUtils.LOGIN_CONTEXT_PACEMAKER_SERVER,
authorizedUsers));
- }
- catch (IOException e) {
+ } catch (IOException e) {
throw new RuntimeException(e);
}
- }
- else if(authMethod == AuthMethod.NONE) {
+ } else if (authMethod == AuthMethod.NONE) {
LOG.debug("Not authenticating any clients. AuthMethod is NONE");
}
@@ -99,4 +84,10 @@ public class ThriftNettyServerCodec {
}
};
}
+
+ public enum AuthMethod {
+ DIGEST,
+ KERBEROS,
+ NONE
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java b/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java
index c5a5ee5..bc08579 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/Cluster.java
@@ -29,7 +29,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Collectors;
-
import org.apache.storm.Config;
import org.apache.storm.Constants;
import org.apache.storm.daemon.nimbus.TopologyResources;
@@ -49,40 +48,33 @@ import org.slf4j.LoggerFactory;
public class Cluster implements ISchedulingState {
private static final Logger LOG = LoggerFactory.getLogger(Cluster.class);
- private SchedulerAssignmentImpl assignment;
-
/**
* key: supervisor id, value: supervisor details.
*/
private final Map<String, SupervisorDetails> supervisors = new HashMap<>();
-
/**
* key: rack, value: nodes in that rack.
*/
private final Map<String, List<String>> networkTopography = new HashMap<>();
-
/**
* key: topologyId, value: topology's current assignments.
*/
private final Map<String, SchedulerAssignmentImpl> assignments = new HashMap<>();
-
/**
* key topologyId, Value: scheduler's status.
*/
private final Map<String, String> status = new HashMap<>();
-
/**
* A map from hostname to supervisor ids.
*/
private final Map<String, List<String>> hostToId = new HashMap<>();
-
private final Map<String, Object> conf;
-
- private Set<String> blackListedHosts = new HashSet<>();
- private INimbus inimbus;
private final Topologies topologies;
private final Map<String, Map<WorkerSlot, NormalizedResourceRequest>> nodeToScheduledResourcesCache;
private final Map<String, Set<WorkerSlot>> nodeToUsedSlotsCache;
+ private SchedulerAssignmentImpl assignment;
+ private Set<String> blackListedHosts = new HashSet<>();
+ private INimbus inimbus;
public Cluster(
INimbus nimbus,
@@ -111,7 +103,7 @@ public class Cluster implements ISchedulingState {
/**
* Testing Constructor that takes an existing cluster and replaces the topologies in it.
*
- * @param src the original cluster
+ * @param src the original cluster
* @param topologies the new topolgoies to use
*/
@VisibleForTesting
@@ -195,8 +187,61 @@ public class Cluster implements ISchedulingState {
}
/**
- * Check if the given topology is allowed for modification right now. If not throw an
- * IllegalArgumentException else go on.
+ * Get heap memory usage for a worker's main process and logwriter process.
+ *
+ * @param topConf - the topology config
+ * @return the assigned memory (in MB)
+ */
+ public static double getAssignedMemoryForSlot(final Map<String, Object> topConf) {
+ double totalWorkerMemory = 0.0;
+ final Integer topologyWorkerDefaultMemoryAllocation = 768;
+
+ List<String> topologyWorkerGcChildopts = ConfigUtils.getValueAsList(
+ Config.TOPOLOGY_WORKER_GC_CHILDOPTS, topConf);
+ List<String> workerGcChildopts = ConfigUtils.getValueAsList(
+ Config.WORKER_GC_CHILDOPTS, topConf);
+ Double memGcChildopts = null;
+ memGcChildopts = Utils.parseJvmHeapMemByChildOpts(
+ topologyWorkerGcChildopts, null);
+ if (memGcChildopts == null) {
+ memGcChildopts = Utils.parseJvmHeapMemByChildOpts(
+ workerGcChildopts, null);
+ }
+
+ List<String> topologyWorkerChildopts = ConfigUtils.getValueAsList(
+ Config.TOPOLOGY_WORKER_CHILDOPTS, topConf);
+ Double memTopologyWorkerChildopts = Utils.parseJvmHeapMemByChildOpts(
+ topologyWorkerChildopts, null);
+
+ List<String> workerChildopts = ConfigUtils.getValueAsList(
+ Config.WORKER_CHILDOPTS, topConf);
+ Double memWorkerChildopts = Utils.parseJvmHeapMemByChildOpts(
+ workerChildopts, null);
+
+ if (memGcChildopts != null) {
+ totalWorkerMemory += memGcChildopts;
+ } else if (memTopologyWorkerChildopts != null) {
+ totalWorkerMemory += memTopologyWorkerChildopts;
+ } else if (memWorkerChildopts != null) {
+ totalWorkerMemory += memWorkerChildopts;
+ } else {
+ Object workerHeapMemoryMb = topConf.get(
+ Config.WORKER_HEAP_MEMORY_MB);
+ totalWorkerMemory += ObjectReader.getInt(
+ workerHeapMemoryMb, topologyWorkerDefaultMemoryAllocation);
+ }
+
+ List<String> topoWorkerLwChildopts = ConfigUtils.getValueAsList(
+ Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS, topConf);
+ if (topoWorkerLwChildopts != null) {
+ totalWorkerMemory += Utils.parseJvmHeapMemByChildOpts(
+ topoWorkerLwChildopts, 0.0);
+ }
+ return totalWorkerMemory;
+ }
+
+ /**
+ * Check if the given topology is allowed for modification right now. If not throw an IllegalArgumentException else go on.
*
* @param topologyId the id of the topology to check
*/
@@ -204,6 +249,16 @@ public class Cluster implements ISchedulingState {
//NOOP
}
+ @Override
+ public Topologies getTopologies() {
+ return topologies;
+ }
+
+ @Override
+ public Set<String> getBlacklistedHosts() {
+ return blackListedHosts;
+ }
+
/**
* Set the list of hosts that are blacklisted.
*
@@ -218,16 +273,6 @@ public class Cluster implements ISchedulingState {
blackListedHosts.addAll(hosts);
}
- @Override
- public Topologies getTopologies() {
- return topologies;
- }
-
- @Override
- public Set<String> getBlacklistedHosts() {
- return blackListedHosts;
- }
-
public void blacklistHost(String host) {
blackListedHosts.add(host);
}
@@ -304,7 +349,8 @@ public class Cluster implements ISchedulingState {
@Override
public Set<Integer> getUsedPorts(SupervisorDetails supervisor) {
- return nodeToUsedSlotsCache.computeIfAbsent(supervisor.getId(), (x) -> new HashSet<>()).stream().map(WorkerSlot::getPort).collect(Collectors.toSet());
+ return nodeToUsedSlotsCache.computeIfAbsent(supervisor.getId(), (x) -> new HashSet<>()).stream().map(WorkerSlot::getPort)
+ .collect(Collectors.toSet());
}
@Override
@@ -424,12 +470,12 @@ public class Cluster implements ISchedulingState {
totalResources.addOnHeap(shared.get_off_heap_worker());
addResource(
- sharedTotalResources,
- Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, shared.get_off_heap_worker()
+ sharedTotalResources,
+ Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, shared.get_off_heap_worker()
);
addResource(
- sharedTotalResources,
- Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, shared.get_on_heap()
+ sharedTotalResources,
+ Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, shared.get_on_heap()
);
}
sharedTotalResources = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(sharedTotalResources);
@@ -441,12 +487,12 @@ public class Cluster implements ISchedulingState {
ret.set_mem_off_heap(totalResources.getOffHeapMemoryMb());
ret.set_mem_on_heap(totalResources.getOnHeapMemoryMb());
ret.set_shared_mem_off_heap(
- sharedTotalResources.getOrDefault(
- Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, 0.0)
+ sharedTotalResources.getOrDefault(
+ Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, 0.0)
);
ret.set_shared_mem_on_heap(
- sharedTotalResources.getOrDefault(
- Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, 0.0)
+ sharedTotalResources.getOrDefault(
+ Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, 0.0)
);
return ret;
}
@@ -493,22 +539,22 @@ public class Cluster implements ISchedulingState {
if (memoryAdded > memoryAvailable) {
if (LOG.isTraceEnabled()) {
LOG.trace("Could not schedule {}:{} on {} not enough Mem {} > {}",
- td.getName(),
- exec,
- ws,
- memoryAdded,
- memoryAvailable);
+ td.getName(),
+ exec,
+ ws,
+ memoryAdded,
+ memoryAvailable);
}
return false;
}
if (afterOnHeap > maxHeap) {
if (LOG.isTraceEnabled()) {
LOG.trace("Could not schedule {}:{} on {} HEAP would be too large {} > {}",
- td.getName(),
- exec,
- ws,
- afterOnHeap,
- maxHeap);
+ td.getName(),
+ exec,
+ ws,
+ afterOnHeap,
+ maxHeap);
}
return false;
}
@@ -531,9 +577,9 @@ public class Cluster implements ISchedulingState {
if (td == null) {
throw new IllegalArgumentException(
"Trying to schedule for topo "
- + topologyId
- + " but that is not a known topology "
- + topologies.getAllIds());
+ + topologyId
+ + " but that is not a known topology "
+ + topologies.getAllIds());
}
WorkerResources resources = calculateWorkerResources(td, executors);
SchedulerAssignmentImpl assignment = assignments.get(topologyId);
@@ -545,14 +591,14 @@ public class Cluster implements ISchedulingState {
if (assignment.isExecutorAssigned(executor)) {
throw new RuntimeException(
"Attempting to assign executor: "
- + executor
- + " of topology: "
- + topologyId
- + " to workerslot: "
- + slot
- + ". The executor is already assigned to workerslot: "
- + assignment.getExecutorToSlot().get(executor)
- + ". The executor must unassigned before it can be assigned to another slot!");
+ + executor
+ + " of topology: "
+ + topologyId
+ + " to workerslot: "
+ + slot
+ + ". The executor is already assigned to workerslot: "
+ + assignment.getExecutorToSlot().get(executor)
+ + ". The executor must unassigned before it can be assigned to another slot!");
}
}
}
@@ -587,7 +633,7 @@ public class Cluster implements ISchedulingState {
/**
* Calculate the amount of shared off heap memory on a given nodes with the given assignment.
*
- * @param nodeId the id of the node
+ * @param nodeId the id of the node
* @param assignment the current assignment
* @return the amount of shared off heap memory for that node in MB
*/
@@ -759,59 +805,6 @@ public class Cluster implements ISchedulingState {
}
/**
- * Get heap memory usage for a worker's main process and logwriter process.
- * @param topConf - the topology config
- * @return the assigned memory (in MB)
- */
- public static double getAssignedMemoryForSlot(final Map<String, Object> topConf) {
- double totalWorkerMemory = 0.0;
- final Integer topologyWorkerDefaultMemoryAllocation = 768;
-
- List<String> topologyWorkerGcChildopts = ConfigUtils.getValueAsList(
- Config.TOPOLOGY_WORKER_GC_CHILDOPTS, topConf);
- List<String> workerGcChildopts = ConfigUtils.getValueAsList(
- Config.WORKER_GC_CHILDOPTS, topConf);
- Double memGcChildopts = null;
- memGcChildopts = Utils.parseJvmHeapMemByChildOpts(
- topologyWorkerGcChildopts, null);
- if (memGcChildopts == null) {
- memGcChildopts = Utils.parseJvmHeapMemByChildOpts(
- workerGcChildopts, null);
- }
-
- List<String> topologyWorkerChildopts = ConfigUtils.getValueAsList(
- Config.TOPOLOGY_WORKER_CHILDOPTS, topConf);
- Double memTopologyWorkerChildopts = Utils.parseJvmHeapMemByChildOpts(
- topologyWorkerChildopts, null);
-
- List<String> workerChildopts = ConfigUtils.getValueAsList(
- Config.WORKER_CHILDOPTS, topConf);
- Double memWorkerChildopts = Utils.parseJvmHeapMemByChildOpts(
- workerChildopts, null);
-
- if (memGcChildopts != null) {
- totalWorkerMemory += memGcChildopts;
- } else if (memTopologyWorkerChildopts != null) {
- totalWorkerMemory += memTopologyWorkerChildopts;
- } else if (memWorkerChildopts != null) {
- totalWorkerMemory += memWorkerChildopts;
- } else {
- Object workerHeapMemoryMb = topConf.get(
- Config.WORKER_HEAP_MEMORY_MB);
- totalWorkerMemory += ObjectReader.getInt(
- workerHeapMemoryMb, topologyWorkerDefaultMemoryAllocation);
- }
-
- List<String> topoWorkerLwChildopts = ConfigUtils.getValueAsList(
- Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS, topConf);
- if (topoWorkerLwChildopts != null) {
- totalWorkerMemory += Utils.parseJvmHeapMemByChildOpts(
- topoWorkerLwChildopts, 0.0);
- }
- return totalWorkerMemory;
- }
-
- /**
* set scheduler status for a topology.
*/
public void setStatus(TopologyDetails td, String statusMessage) {
@@ -837,10 +830,6 @@ public class Cluster implements ISchedulingState {
return status;
}
- public String getStatus(String topoId) {
- return status.get(topoId);
- }
-
/**
* set scheduler status map.
*/
@@ -858,6 +847,10 @@ public class Cluster implements ISchedulingState {
this.status.putAll(statusMap);
}
+ public String getStatus(String topoId) {
+ return status.get(topoId);
+ }
+
@Override
public Map<String, TopologyResources> getTopologyResourcesMap() {
Map<String, TopologyResources> ret = new HashMap<>(assignments.size());
@@ -925,7 +918,7 @@ public class Cluster implements ISchedulingState {
/**
- * This medhod updates ScheduledResources and UsedSlots cache for given workerSlot
+ * This medhod updates ScheduledResources and UsedSlots cache for given workerSlot
*
* @param workerSlot
* @param workerResources
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/Component.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/Component.java b/storm-server/src/main/java/org/apache/storm/scheduler/Component.java
index 6a0de72..c4a28c3 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/Component.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/Component.java
@@ -21,7 +21,6 @@ package org.apache.storm.scheduler;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-
import org.apache.storm.generated.ComponentType;
public class Component {
@@ -33,9 +32,10 @@ public class Component {
/**
* Create a new component.
- * @param type the type of component this is
+ *
+ * @param type the type of component this is
* @param compId the id of the component
- * @param execs the executors for this component.
+ * @param execs the executors for this component.
*/
public Component(ComponentType type, String compId, List<ExecutorDetails> execs) {
this.type = type;
@@ -76,13 +76,13 @@ public class Component {
@Override
public String toString() {
return "{id: "
- + getId()
- + " Parents: "
- + getParents()
- + " Children: "
- + getChildren()
- + " Execs: "
- + getExecs()
- + "}";
+ + getId()
+ + " Parents: "
+ + getParents()
+ + " Children: "
+ + getChildren()
+ + " Execs: "
+ + getExecs()
+ + "}";
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/DefaultScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/DefaultScheduler.java
index 82630ca..84498a3 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/DefaultScheduler.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/DefaultScheduler.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.scheduler;
import java.util.HashMap;
@@ -23,7 +24,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-
import org.apache.storm.utils.Utils;
public class DefaultScheduler implements IScheduler {
@@ -44,7 +44,7 @@ public class DefaultScheduler implements IScheduler {
}
for (WorkerSlot slot : slots) {
- existingSlots.remove(slot);
+ existingSlots.remove(slot);
}
return existingSlots.keySet();
@@ -74,7 +74,8 @@ public class DefaultScheduler implements IScheduler {
List<WorkerSlot> availableSlots = cluster.getAvailableSlots();
Set<ExecutorDetails> allExecutors = topology.getExecutors();
- Map<WorkerSlot, List<ExecutorDetails>> aliveAssigned = EvenScheduler.getAliveAssignedWorkerSlotExecutors(cluster, topology.getId());
+ Map<WorkerSlot, List<ExecutorDetails>> aliveAssigned =
+ EvenScheduler.getAliveAssignedWorkerSlotExecutors(cluster, topology.getId());
Set<ExecutorDetails> aliveExecutors = new HashSet<ExecutorDetails>();
for (List<ExecutorDetails> list : aliveAssigned.values()) {
aliveExecutors.addAll(list);
@@ -85,7 +86,7 @@ public class DefaultScheduler implements IScheduler {
Set<WorkerSlot> badSlots = null;
if (totalSlotsToUse > aliveAssigned.size() || !allExecutors.equals(aliveExecutors)) {
- badSlots = badSlots(aliveAssigned, allExecutors.size(), totalSlotsToUse);
+ badSlots = badSlots(aliveAssigned, allExecutors.size(), totalSlotsToUse);
}
if (badSlots != null) {
cluster.freeSlots(badSlots);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/EvenScheduler.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/EvenScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/EvenScheduler.java
index 9cbce18..f4ac258 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/EvenScheduler.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/EvenScheduler.java
@@ -20,7 +20,6 @@ package org.apache.storm.scheduler;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
-
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -30,7 +29,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-
import org.apache.storm.utils.ServerUtils;
import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
@@ -116,7 +114,7 @@ public class EvenScheduler implements IScheduler {
//allow requesting slots number bigger than available slots
int toIndex = (totalSlotsToUse - aliveAssigned.size())
- > sortedList.size() ? sortedList.size() : (totalSlotsToUse - aliveAssigned.size());
+ > sortedList.size() ? sortedList.size() : (totalSlotsToUse - aliveAssigned.size());
List<WorkerSlot> reassignSlots = sortedList.subList(0, toIndex);
Set<ExecutorDetails> aliveExecutors = new HashSet<ExecutorDetails>();
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/ExecutorDetails.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/ExecutorDetails.java b/storm-server/src/main/java/org/apache/storm/scheduler/ExecutorDetails.java
index bbbbf3f..855cc96 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/ExecutorDetails.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/ExecutorDetails.java
@@ -40,8 +40,8 @@ public class ExecutorDetails {
if (other == null || !(other instanceof ExecutorDetails)) {
return false;
}
-
- ExecutorDetails executor = (ExecutorDetails)other;
+
+ ExecutorDetails executor = (ExecutorDetails) other;
return (this.startTask == executor.startTask) && (this.endTask == executor.endTask);
}
@@ -49,7 +49,7 @@ public class ExecutorDetails {
public int hashCode() {
return this.startTask + 13 * this.endTask;
}
-
+
@Override
public String toString() {
return "[" + this.startTask + ", " + this.endTask + "]";
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/INimbus.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/INimbus.java b/storm-server/src/main/java/org/apache/storm/scheduler/INimbus.java
index db2fed9..4e26708 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/INimbus.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/INimbus.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler;
@@ -42,6 +36,6 @@ public interface INimbus {
* map from node id to supervisor details.
*/
String getHostName(Map<String, SupervisorDetails> existingSupervisors, String nodeId);
-
- IScheduler getForcedScheduler();
+
+ IScheduler getForcedScheduler();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/IScheduler.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/IScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/IScheduler.java
index 5b79f11..18e9afd 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/IScheduler.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/IScheduler.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler;
@@ -21,9 +15,9 @@ package org.apache.storm.scheduler;
import java.util.Map;
public interface IScheduler {
-
+
void prepare(Map<String, Object> conf);
-
+
/**
* Set assignments for the topologies which needs scheduling. The new assignments is available
* through `cluster.getAssignments()`
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/ISchedulingState.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/ISchedulingState.java b/storm-server/src/main/java/org/apache/storm/scheduler/ISchedulingState.java
index 187a03c..b0d94b5 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/ISchedulingState.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/ISchedulingState.java
@@ -27,17 +27,21 @@ import org.apache.storm.generated.WorkerResources;
import org.apache.storm.scheduler.resource.normalization.NormalizedResourceOffer;
import org.apache.storm.scheduler.resource.normalization.NormalizedResourceRequest;
-/** An interface that provides access to the current scheduling state. */
+/**
+ * An interface that provides access to the current scheduling state.
+ */
public interface ISchedulingState {
/**
* Get all of the topologies.
+ *
* @return all of the topologies that are a part of the cluster.
*/
Topologies getTopologies();
/**
* Get all of the topologies that need scheduling.
+ *
* @return all of the topologies that are not fully scheduled.
*/
List<TopologyDetails> needsSchedulingTopologies();
@@ -48,16 +52,16 @@ public interface ISchedulingState {
* <p>A topology needs scheduling if one of the following conditions holds:
*
* <ul>
- * <li>Although the topology is assigned slots, but is squeezed. i.e. the topology is assigned
- * less slots than desired.
- * <li>There are unassigned executors in this topology
+ * <li>Although the topology is assigned slots, but is squeezed. i.e. the topology is assigned
+ * less slots than desired.
+ * <li>There are unassigned executors in this topology
* </ul>
*/
boolean needsScheduling(TopologyDetails topology);
/**
- * Like {@link #needsScheduling(TopologyDetails)} but does not take into account the number of
- * workers requested. This is because the number of workers is ignored in RAS
+ * Like {@link #needsScheduling(TopologyDetails)} but does not take into account the number of workers requested. This is because the
+ * number of workers is ignored in RAS
*
* @param topology the topology to check
* @return true if the topology needs scheduling else false.
@@ -66,6 +70,7 @@ public interface ISchedulingState {
/**
* Get all of the hosts that are blacklisted.
+ *
* @return all of the hosts that are blacklisted
*/
Set<String> getBlacklistedHosts();
@@ -104,6 +109,7 @@ public interface ISchedulingState {
/**
* Get the executor to component name map for executors that need to be scheduled.
+ *
* @param topology the topology this is for
* @return a executor -> component-id map which needs scheduling in this topology.
*/
@@ -111,16 +117,21 @@ public interface ISchedulingState {
/**
* Get the component name to executor list for executors that need to be scheduled.
+ *
* @param topology the topology this is for
* @return a component-id -> executors map which needs scheduling in this topology.
*/
Map<String, List<ExecutorDetails>> getNeedsSchedulingComponentToExecutors(
TopologyDetails topology);
- /** Get all the used ports of this supervisor. */
+ /**
+ * Get all the used ports of this supervisor.
+ */
Set<Integer> getUsedPorts(SupervisorDetails supervisor);
- /** Return the available ports of this supervisor. */
+ /**
+ * Return the available ports of this supervisor.
+ */
Set<Integer> getAvailablePorts(SupervisorDetails supervisor);
/**
@@ -131,10 +142,14 @@ public interface ISchedulingState {
*/
Set<Integer> getAssignablePorts(SupervisorDetails supervisor);
- /** Return all the available slots on this supervisor. */
+ /**
+ * Return all the available slots on this supervisor.
+ */
List<WorkerSlot> getAvailableSlots(SupervisorDetails supervisor);
- /** Get all the available worker slots in the cluster. */
+ /**
+ * Get all the available worker slots in the cluster.
+ */
List<WorkerSlot> getAvailableSlots();
/**
@@ -145,14 +160,19 @@ public interface ISchedulingState {
*/
List<WorkerSlot> getAssignableSlots(SupervisorDetails supervisor);
- /** Get all non-blacklisted slots in the cluster. */
+ /**
+ * Get all non-blacklisted slots in the cluster.
+ */
List<WorkerSlot> getAssignableSlots();
- /** Get all currently occupied slots. */
+ /**
+ * Get all currently occupied slots.
+ */
Collection<WorkerSlot> getUsedSlots();
/**
* Check if a slot is occupied or not.
+ *
* @param slot the slot be to checked.
* @return true if the specified slot is occupied.
*/
@@ -160,20 +180,20 @@ public interface ISchedulingState {
/**
* Get the number of workers assigned to a topology.
+ *
* @param topology the topology this is for
* @return the number of workers assigned to this topology.
*/
int getAssignedNumWorkers(TopologyDetails topology);
/**
- * Would scheduling exec on ws fit? With a heap <= maxHeap total memory added <= memoryAvailable
- * and cpu added <= cpuAvailable.
+ * Would scheduling exec on ws fit? With a heap <= maxHeap total memory added <= memoryAvailable and cpu added <= cpuAvailable.
*
- * @param ws the slot to put it in
- * @param exec the executor to investigate
- * @param td the topology detains for this executor
+ * @param ws the slot to put it in
+ * @param exec the executor to investigate
+ * @param td the topology detains for this executor
* @param resourcesAvailable all the available resources
- * @param maxHeap the maximum heap size for ws
+ * @param maxHeap the maximum heap size for ws
* @return true it fits else false
*/
boolean wouldFit(
@@ -183,13 +203,19 @@ public interface ISchedulingState {
NormalizedResourceOffer resourcesAvailable,
double maxHeap);
- /** get the current assignment for the topology. */
+ /**
+ * get the current assignment for the topology.
+ */
SchedulerAssignment getAssignmentById(String topologyId);
- /** get slots used by a topology. */
+ /**
+ * get slots used by a topology.
+ */
Collection<WorkerSlot> getUsedSlotsByTopologyId(String topologyId);
- /** Get a specific supervisor with the <code>nodeId</code>. */
+ /**
+ * Get a specific supervisor with the <code>nodeId</code>.
+ */
SupervisorDetails getSupervisorById(String nodeId);
/**
@@ -200,45 +226,55 @@ public interface ISchedulingState {
*/
List<SupervisorDetails> getSupervisorsByHost(String host);
- /** Get all the assignments. */
+ /**
+ * Get all the assignments.
+ */
Map<String, SchedulerAssignment> getAssignments();
- /** Get all the supervisors. */
+ /**
+ * Get all the supervisors.
+ */
Map<String, SupervisorDetails> getSupervisors();
- /** Get all scheduled resources for node. **/
+ /**
+ * Get all scheduled resources for node.
+ **/
NormalizedResourceRequest getAllScheduledResourcesForNode(String nodeId);
- /** Get the total amount of CPU resources in cluster. */
+ /**
+ * Get the total amount of CPU resources in cluster.
+ */
double getClusterTotalCpuResource();
- /** Get the total amount of memory resources in cluster. */
+ /**
+ * Get the total amount of memory resources in cluster.
+ */
double getClusterTotalMemoryResource();
- /** Get the network topography (rackId -> nodes in the rack). */
+ /**
+ * Get the network topography (rackId -> nodes in the rack).
+ */
Map<String, List<String>> getNetworkTopography();
- /** Get all topology scheduler statuses. */
+ /**
+ * Get all topology scheduler statuses.
+ */
Map<String, String> getStatusMap();
/**
- * Get the amount of resources used by topologies. Used for displaying resource information on the
- * UI.
+ * Get the amount of resources used by topologies. Used for displaying resource information on the UI.
*
- * @return a map that contains multiple topologies and the resources the topology requested and
- * assigned. Key: topology id Value: an array that describes the resources the topology
- * requested and assigned in the following format: {requestedMemOnHeap, requestedMemOffHeap,
- * requestedCpu, assignedMemOnHeap, assignedMemOffHeap, assignedCpu}
+ * @return a map that contains multiple topologies and the resources the topology requested and assigned. Key: topology id Value: an
+ * array that describes the resources the topology requested and assigned in the following format: {requestedMemOnHeap,
+ * requestedMemOffHeap, requestedCpu, assignedMemOnHeap, assignedMemOffHeap, assignedCpu}
*/
Map<String, TopologyResources> getTopologyResourcesMap();
/**
- * Get the amount of used and free resources on a supervisor. Used for displaying resource
- * information on the UI
+ * Get the amount of used and free resources on a supervisor. Used for displaying resource information on the UI
*
- * @return a map where the key is the supervisor id and the value is a map that represents
- * resource usage for a supervisor in the following format: {totalMem, totalCpu, usedMem,
- * usedCpu}
+ * @return a map where the key is the supervisor id and the value is a map that represents resource usage for a supervisor in the
+ * following format: {totalMem, totalCpu, usedMem, usedCpu}
*/
Map<String, SupervisorResources> getSupervisorsResourcesMap();
@@ -273,6 +309,8 @@ public interface ISchedulingState {
*/
double getScheduledCpuForNode(String nodeId);
- /** Get the nimbus configuration. */
+ /**
+ * Get the nimbus configuration.
+ */
Map<String, Object> getConf();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/ISupervisor.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/ISupervisor.java b/storm-server/src/main/java/org/apache/storm/scheduler/ISupervisor.java
index 4ecf28e..08d1229 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/ISupervisor.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/ISupervisor.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler;
@@ -25,6 +19,7 @@ public interface ISupervisor {
void prepare(Map<String, Object> topoConf, String schedulerLocalDir);
// for mesos, this is {hostname}-{topologyid}
+
/**
* The id used for writing metadata into ZK.
*/
@@ -39,7 +34,7 @@ public interface ISupervisor {
String getAssignmentId();
Object getMetadata();
-
+
boolean confirmAssigned(int port);
// calls this before actually killing the worker locally...
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/IsolationScheduler.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/IsolationScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/IsolationScheduler.java
index 3201a40..03c0c6a 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/IsolationScheduler.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/IsolationScheduler.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.scheduler;
import java.util.ArrayList;
@@ -28,7 +23,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-
import org.apache.commons.lang.Validate;
import org.apache.storm.DaemonConfig;
import org.apache.storm.utils.Utils;
@@ -63,7 +57,8 @@ public class IsolationScheduler implements IScheduler {
// get host -> all assignable worker slots for non-blacklisted machines (assigned or not assigned)
// will then have a list of machines that need to be assigned (machine -> [topology, list of list of executors])
- // match each spec to a machine (who has the right number of workers), free everything else on that machine and assign those slots (do one topology at a time)
+ // match each spec to a machine (who has the right number of workers), free everything else on that machine and assign those slots
+ // (do one topology at a time)
// blacklist all machines who had production slots defined
// log isolated topologies who weren't able to get enough slots / machines
// run default scheduler on isolated topologies that didn't have enough slots + non-isolated topologies on remaining machines
@@ -85,9 +80,9 @@ public class IsolationScheduler implements IScheduler {
int numWorkers = assignments.size();
if (isoIds.contains(topologyId)
- && checkAssignmentTopology(assignments, topologyId)
- && distribution.containsKey(numWorkers)
- && checkAssignmentWorkerSpecs(assignments, workerSpecs)) {
+ && checkAssignmentTopology(assignments, topologyId)
+ && distribution.containsKey(numWorkers)
+ && checkAssignmentWorkerSpecs(assignments, workerSpecs)) {
decrementDistribution(distribution, numWorkers);
for (AssignmentInfo ass : assignments) {
workerSpecs.remove(ass.getExecutors());
@@ -127,9 +122,9 @@ public class IsolationScheduler implements IScheduler {
List<String> failedTopologyIds = extractFailedTopologyIds(topologyWorkerSpecs);
if (failedTopologyIds.size() > 0) {
LOG.warn("Unable to isolate topologies " + failedTopologyIds
- + ". No machine had enough worker slots to run the remaining workers for these topologies. "
- + "Clearing all other resources and will wait for enough resources for "
- + "isolated topologies before allocating any other resources.");
+ + ". No machine had enough worker slots to run the remaining workers for these topologies. "
+ + "Clearing all other resources and will wait for enough resources for "
+ + "isolated topologies before allocating any other resources.");
// clear workers off all hosts that are not blacklisted
Map<String, Set<WorkerSlot>> usedSlots = hostToUsedSlots(cluster);
Set<Map.Entry<String, Set<WorkerSlot>>> entries = usedSlots.entrySet();
@@ -176,7 +171,7 @@ public class IsolationScheduler implements IScheduler {
private List<String> extractFailedTopologyIds(Map<String, Set<Set<ExecutorDetails>>> isoTopologyWorkerSpecs) {
List<String> failedTopologyIds = new ArrayList<String>();
- for (Map.Entry<String, Set<Set<ExecutorDetails>>> topoWorkerSpecsEntry : isoTopologyWorkerSpecs.entrySet()){
+ for (Map.Entry<String, Set<Set<ExecutorDetails>>> topoWorkerSpecsEntry : isoTopologyWorkerSpecs.entrySet()) {
Set<Set<ExecutorDetails>> workerSpecs = topoWorkerSpecsEntry.getValue();
if (workerSpecs != null && !workerSpecs.isEmpty()) {
failedTopologyIds.add(topoWorkerSpecsEntry.getKey());
@@ -195,7 +190,7 @@ public class IsolationScheduler implements IScheduler {
}
private Map<String, List<AssignmentInfo>> hostAssignments(Cluster cluster) {
- Collection<SchedulerAssignment> assignmentValues = cluster.getAssignments().values();
+ Collection<SchedulerAssignment> assignmentValues = cluster.getAssignments().values();
Map<String, List<AssignmentInfo>> hostAssignments = new HashMap<String, List<AssignmentInfo>>();
for (SchedulerAssignment sa : assignmentValues) {
@@ -237,7 +232,7 @@ public class IsolationScheduler implements IScheduler {
bucketExecutors.put(bucketIndex, executors);
}
executors.add(executor);
- bucketIndex = (bucketIndex+1) % numWorkers;
+ bucketIndex = (bucketIndex + 1) % numWorkers;
}
return new HashSet<Set<ExecutorDetails>>(bucketExecutors.values());
@@ -325,11 +320,11 @@ public class IsolationScheduler implements IScheduler {
sortHostAssignSlots.add(new HostAssignableSlots(entry.getKey(), entry.getValue()));
}
Collections.sort(sortHostAssignSlots, new Comparator<HostAssignableSlots>() {
- @Override
- public int compare(HostAssignableSlots o1, HostAssignableSlots o2) {
- return o2.getWorkerSlots().size() - o1.getWorkerSlots().size();
- }
- });
+ @Override
+ public int compare(HostAssignableSlots o1, HostAssignableSlots o2) {
+ return o2.getWorkerSlots().size() - o1.getWorkerSlots().size();
+ }
+ });
Collections.shuffle(sortHostAssignSlots);
return new LinkedList<HostAssignableSlots>(sortHostAssignSlots);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignment.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignment.java b/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignment.java
index c16a67f..347c95f 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignment.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignment.java
@@ -21,13 +21,12 @@ package org.apache.storm.scheduler;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
-
-import org.apache.storm.generated.Assignment;
import org.apache.storm.generated.WorkerResources;
public interface SchedulerAssignment {
/**
* Is this slot part of this assignment or not.
+ *
* @param slot the slot to check.
* @return true if the slot is occupied by this assignment else false.
*/
@@ -35,50 +34,57 @@ public interface SchedulerAssignment {
/**
* Is the executor assigned or not.
- *
+ *
* @param executor the executor to check it if is assigned.
* @return true if it is assigned else false
*/
public boolean isExecutorAssigned(ExecutorDetails executor);
-
+
/**
* Return the ID of the topology.
+ *
* @return the topology-id this assignment is for.
*/
public String getTopologyId();
/**
* Get the map of executor to WorkerSlot.
+ *
* @return the executor -> slot map.
*/
public Map<ExecutorDetails, WorkerSlot> getExecutorToSlot();
/**
* Get the set of all executors.
- * @return the executors covered by this assignments
+ *
+ * @return the executors covered by this assignments
*/
public Set<ExecutorDetails> getExecutors();
/**
* Get the set of all slots that are a part of this.
+ *
* @return the set of all slots.
*/
public Set<WorkerSlot> getSlots();
/**
* Get the mapping of slot to executors on that slot.
+ *
* @return the slot to the executors assigned to that slot.
*/
public Map<WorkerSlot, Collection<ExecutorDetails>> getSlotToExecutors();
-
+
/**
* Get the slot to resource mapping.
+ *
* @return The slot to resource mapping
*/
public Map<WorkerSlot, WorkerResources> getScheduledResources();
-
+
/**
* Get the total shared off heap memory mapping.
+ *
* @return host to total shared off heap memory mapping.
*/
public Map<String, Double> getNodeIdToTotalSharedOffHeapMemory();
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignmentImpl.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignmentImpl.java b/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignmentImpl.java
index f92c3d1..dcb2ebc 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignmentImpl.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/SchedulerAssignmentImpl.java
@@ -26,7 +26,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
import org.apache.storm.generated.WorkerResources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,14 +49,15 @@ public class SchedulerAssignmentImpl implements SchedulerAssignment {
/**
* Create a new assignment.
- * @param topologyId the id of the topology the assignment is for.
- * @param executorToSlot the executor to slot mapping for the assignment. Can be null and set through other methods later.
- * @param resources the resources for the current assignments. Can be null and set through other methods later.
+ *
+ * @param topologyId the id of the topology the assignment is for.
+ * @param executorToSlot the executor to slot mapping for the assignment. Can be null and set through other methods later.
+ * @param resources the resources for the current assignments. Can be null and set through other methods later.
* @param nodeIdToTotalSharedOffHeap the shared memory for this assignment can be null and set through other methods later.
*/
public SchedulerAssignmentImpl(String topologyId, Map<ExecutorDetails, WorkerSlot> executorToSlot,
- Map<WorkerSlot, WorkerResources> resources, Map<String, Double> nodeIdToTotalSharedOffHeap) {
- this.topologyId = topologyId;
+ Map<WorkerSlot, WorkerResources> resources, Map<String, Double> nodeIdToTotalSharedOffHeap) {
+ this.topologyId = topologyId;
if (executorToSlot != null) {
if (executorToSlot.entrySet().stream().anyMatch((entry) -> entry.getKey() == null || entry.getValue() == null)) {
throw new RuntimeException("Cannot create a scheduling with a null in it " + executorToSlot);
@@ -83,8 +83,8 @@ public class SchedulerAssignmentImpl implements SchedulerAssignment {
}
public SchedulerAssignmentImpl(SchedulerAssignment assignment) {
- this(assignment.getTopologyId(), assignment.getExecutorToSlot(),
- assignment.getScheduledResources(), assignment.getNodeIdToTotalSharedOffHeapMemory());
+ this(assignment.getTopologyId(), assignment.getExecutorToSlot(),
+ assignment.getScheduledResources(), assignment.getNodeIdToTotalSharedOffHeapMemory());
}
@Override
@@ -94,6 +94,7 @@ public class SchedulerAssignmentImpl implements SchedulerAssignment {
/**
* Like the equals command, but ignores the resources.
+ *
* @param other the object to check for equality against.
* @return true if they are equal, ignoring resources, else false.
*/
@@ -105,11 +106,11 @@ public class SchedulerAssignmentImpl implements SchedulerAssignment {
return false;
}
SchedulerAssignmentImpl o = (SchedulerAssignmentImpl) other;
-
+
return topologyId.equals(o.topologyId)
- && executorToSlot.equals(o.executorToSlot);
+ && executorToSlot.equals(o.executorToSlot);
}
-
+
@Override
public int hashCode() {
final int prime = 31;
@@ -118,7 +119,7 @@ public class SchedulerAssignmentImpl implements SchedulerAssignment {
result = prime * result + ((executorToSlot == null) ? 0 : executorToSlot.hashCode());
return result;
}
-
+
@Override
public boolean equals(Object other) {
if (!equalsIgnoreResources(other)) {
@@ -127,13 +128,13 @@ public class SchedulerAssignmentImpl implements SchedulerAssignment {
SchedulerAssignmentImpl o = (SchedulerAssignmentImpl) other;
return resources.equals(o.resources)
- && nodeIdToTotalSharedOffHeap.equals(o.nodeIdToTotalSharedOffHeap);
+ && nodeIdToTotalSharedOffHeap.equals(o.nodeIdToTotalSharedOffHeap);
}
-
+
@Override
public Set<WorkerSlot> getSlots() {
return new HashSet<>(executorToSlot.values());
- }
+ }
@Deprecated
public void assign(WorkerSlot slot, Collection<ExecutorDetails> executors) {
@@ -180,7 +181,7 @@ public class SchedulerAssignmentImpl implements SchedulerAssignment {
String node = slot.getNodeId();
boolean isFound = false;
- for (WorkerSlot ws: executorToSlot.values()) {
+ for (WorkerSlot ws : executorToSlot.values()) {
if (node.equals(ws.getNodeId())) {
isFound = true;
break;
@@ -243,7 +244,7 @@ public class SchedulerAssignmentImpl implements SchedulerAssignment {
public void setTotalSharedOffHeapMemory(String node, double value) {
nodeIdToTotalSharedOffHeap.put(node, value);
}
-
+
@Override
public Map<String, Double> getNodeIdToTotalSharedOffHeapMemory() {
return nodeIdToTotalSharedOffHeap;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/SingleTopologyCluster.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/SingleTopologyCluster.java b/storm-server/src/main/java/org/apache/storm/scheduler/SingleTopologyCluster.java
index ee8ef00..ff020a4 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/SingleTopologyCluster.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/SingleTopologyCluster.java
@@ -27,7 +27,7 @@ public class SingleTopologyCluster extends Cluster {
/**
* Create a new cluster that only allows modifications to a single topology.
*
- * @param other the current cluster to base this off of
+ * @param other the current cluster to base this off of
* @param topologyId the topology that is allowed to be modified.
*/
public SingleTopologyCluster(Cluster other, String topologyId) {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorDetails.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorDetails.java b/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorDetails.java
index 00b8ea0..4ac1057 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorDetails.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorDetails.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler;
@@ -22,7 +16,6 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import org.apache.storm.Constants;
import org.apache.storm.scheduler.resource.normalization.NormalizedResourceOffer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,13 +38,13 @@ public class SupervisorDetails {
*/
private final Object schedulerMeta;
/**
- * all the ports of the supervisor.
- */
- private Set<Integer> allPorts;
- /**
* Map containing a manifest of resources for the node the supervisor resides.
*/
private final NormalizedResourceOffer totalResources;
+ /**
+ * all the ports of the supervisor.
+ */
+ private Set<Integer> allPorts;
/**
* Create the details of a new supervisor.
@@ -105,20 +98,11 @@ public class SupervisorDetails {
Collection<? extends Number> allPorts, Map<String, Double> totalResources) {
this(id, serverPort, host, null, schedulerMeta, allPorts, totalResources);
}
-
+
@Override
public String toString() {
return getClass().getSimpleName() + " ID: " + id + " HOST: " + host + " META: " + meta
- + " SCHED_META: " + schedulerMeta + " PORTS: " + allPorts;
- }
-
- private void setAllPorts(Collection<? extends Number> allPorts) {
- this.allPorts = new HashSet<>();
- if (allPorts != null) {
- for (Number n: allPorts) {
- this.allPorts.add(n.intValue());
- }
- }
+ + " SCHED_META: " + schedulerMeta + " PORTS: " + allPorts;
}
public String getId() {
@@ -136,11 +120,20 @@ public class SupervisorDetails {
public Object getMeta() {
return meta;
}
-
+
public Set<Integer> getAllPorts() {
return allPorts;
}
+ private void setAllPorts(Collection<? extends Number> allPorts) {
+ this.allPorts = new HashSet<>();
+ if (allPorts != null) {
+ for (Number n : allPorts) {
+ this.allPorts.add(n.intValue());
+ }
+ }
+ }
+
public Object getSchedulerMeta() {
return this.schedulerMeta;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorResources.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorResources.java b/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorResources.java
index ac2a506..8ee686f 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorResources.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/SupervisorResources.java
@@ -31,8 +31,8 @@ public class SupervisorResources {
*
* @param totalMem the total mem on the supervisor
* @param totalCpu the total CPU on the supervisor
- * @param usedMem the used mem on the supervisor
- * @param usedCpu the used CPU on the supervisor
+ * @param usedMem the used mem on the supervisor
+ * @param usedCpu the used CPU on the supervisor
*/
public SupervisorResources(double totalMem, double totalCpu, double usedMem, double usedCpu) {
this.totalMem = totalMem;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/Topologies.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/Topologies.java b/storm-server/src/main/java/org/apache/storm/scheduler/Topologies.java
index 16d230a..b222916 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/Topologies.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/Topologies.java
@@ -30,19 +30,9 @@ public class Topologies implements Iterable<TopologyDetails> {
Map<String, String> nameToId;
Map<String, Map<String, Component>> allComponents;
- private static Map<String, TopologyDetails> mkMap(TopologyDetails[] details) {
- Map<String, TopologyDetails> ret = new HashMap<>();
- for (TopologyDetails td : details) {
- if (ret.put(td.getId(), td) != null) {
- throw new IllegalArgumentException(
- "Cannot have multiple topologies with the id " + td.getId());
- }
- }
- return ret;
- }
-
/**
* Create a new Topologies from a list of TopologyDetails.
+ *
* @param details the list of details to use.
* @throws IllegalArgumentException if duplicate topology ids are found.
*/
@@ -52,6 +42,7 @@ public class Topologies implements Iterable<TopologyDetails> {
/**
* Create a new Topologies from a map of id to topology
+ *
* @param topologies a map of topology id to topology details.
*/
public Topologies(Map<String, TopologyDetails> topologies) {
@@ -74,12 +65,24 @@ public class Topologies implements Iterable<TopologyDetails> {
this(src.topologies);
}
+ private static Map<String, TopologyDetails> mkMap(TopologyDetails[] details) {
+ Map<String, TopologyDetails> ret = new HashMap<>();
+ for (TopologyDetails td : details) {
+ if (ret.put(td.getId(), td) != null) {
+ throw new IllegalArgumentException(
+ "Cannot have multiple topologies with the id " + td.getId());
+ }
+ }
+ return ret;
+ }
+
public Collection<String> getAllIds() {
return topologies.keySet();
}
/**
* Get a topology given an ID
+ *
* @param topologyId the id of the topology to get
* @return the topology or null if it is not found.
*/
@@ -88,8 +91,8 @@ public class Topologies implements Iterable<TopologyDetails> {
}
/**
- * Get a topology given a topology name. Nimbus prevents multiple topologies
- * from having the same name, so this assumes it is true.
+ * Get a topology given a topology name. Nimbus prevents multiple topologies from having the same name, so this assumes it is true.
+ *
* @param topologyName the name of the topology to look for
* @return the a topology with the given name.
*/
@@ -109,6 +112,7 @@ public class Topologies implements Iterable<TopologyDetails> {
/**
* Get all topologies submitted/owned by a given user.
+ *
* @param user the name of the user
* @return all of the topologies submitted by this user.
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java b/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
index 41e7edf..c43f94a 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/TopologyDetails.java
@@ -43,31 +43,30 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TopologyDetails {
+ private static final Logger LOG = LoggerFactory.getLogger(TopologyDetails.class);
private final String topologyId;
private final Map<String, Object> topologyConf;
private final StormTopology topology;
private final Map<ExecutorDetails, String> executorToComponent;
private final int numWorkers;
+ //when topology was launched
+ private final int launchTime;
+ private final String owner;
+ private final String topoName;
//<ExecutorDetails - Task, Map<String - Type of resource, Map<String - type of that resource, Double - amount>>>
private Map<ExecutorDetails, NormalizedResourceRequest> resourceList;
//Max heap size for a worker used by topology
private Double topologyWorkerMaxHeapSize;
//topology priority
private Integer topologyPriority;
- //when topology was launched
- private final int launchTime;
- private final String owner;
- private final String topoName;
-
- private static final Logger LOG = LoggerFactory.getLogger(TopologyDetails.class);
public TopologyDetails(String topologyId, Map<String, Object> topologyConf, StormTopology topology, int numWorkers, String owner) {
- this(topologyId, topologyConf, topology, numWorkers, null, 0, owner);
+ this(topologyId, topologyConf, topology, numWorkers, null, 0, owner);
}
public TopologyDetails(String topologyId, Map<String, Object> topologyConf, StormTopology topology,
int numWorkers, Map<ExecutorDetails, String> executorToComponents, String owner) {
- this(topologyId, topologyConf, topology, numWorkers, executorToComponents, 0, owner);
+ this(topologyId, topologyConf, topology, numWorkers, executorToComponents, 0, owner);
}
public TopologyDetails(String topologyId, Map<String, Object> topologyConf, StormTopology topology, int numWorkers,
@@ -168,8 +167,8 @@ public class TopologyDetails {
getExecutorToComponent().get(exec),
exec,
topologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB),
- resourceList.get(exec)
- );
+ resourceList.get(exec)
+ );
addDefaultResforExec(exec);
}
}
@@ -194,9 +193,8 @@ public class TopologyDetails {
}
/**
- * Returns a representation of the non-system components of the topology graph Each Component
- * object in the returning map is populated with the list of its parents, children and execs
- * assigned to that component.
+ * Returns a representation of the non-system components of the topology graph Each Component object in the returning map is populated
+ * with the list of its parents, children and execs assigned to that component.
*
* @return a map of components
*/
@@ -248,6 +246,7 @@ public class TopologyDetails {
/**
* Gets the on heap memory requirement for a certain task within a topology.
+ *
* @param exec the executor the inquiry is concerning.
* @return Double the amount of on heap memory requirement for this exec in topology topoId.
*/
@@ -255,13 +254,15 @@ public class TopologyDetails {
Double ret = null;
if (hasExecInTopo(exec)) {
ret = resourceList
- .get(exec).getOnHeapMemoryMb();;
+ .get(exec).getOnHeapMemoryMb();
+ ;
}
return ret;
}
/**
* Gets the off heap memory requirement for a certain task within a topology.
+ *
* @param exec the executor the inquiry is concerning.
* @return Double the amount of off heap memory requirement for this exec in topology topoId.
*/
@@ -269,32 +270,33 @@ public class TopologyDetails {
Double ret = null;
if (hasExecInTopo(exec)) {
ret = resourceList
- .get(exec).getOffHeapMemoryMb();
+ .get(exec).getOffHeapMemoryMb();
}
return ret;
}
/**
* Gets the total memory requirement for a task.
+ *
* @param exec the executor the inquiry is concerning.
* @return Double the total memory requirement for this exec in topology topoId.
*/
public Double getTotalMemReqTask(ExecutorDetails exec) {
if (hasExecInTopo(exec)) {
return getOffHeapMemoryRequirement(exec)
- + getOnHeapMemoryRequirement(exec);
+ + getOnHeapMemoryRequirement(exec);
}
return null;
}
/**
* Gets the total memory resource list for a set of tasks that is part of a topology.
+ *
* @param executors all executors for a topology
- * @return Map<ExecutorDetails, Double> ,
- * a map of the total memory requirement for all tasks in topology topoId.
+ * @return Map<ExecutorDetails , Double> , a map of the total memory requirement for all tasks in topology topoId.
*/
public Set<SharedMemory> getSharedMemoryRequests(
- Collection<ExecutorDetails> executors
+ Collection<ExecutorDetails> executors
) {
Set<String> components = new HashSet<>();
for (ExecutorDetails exec : executors) {
@@ -324,6 +326,7 @@ public class TopologyDetails {
/**
* Get the total resource requirement for an executor.
+ *
* @param exec the executor to get the resources for.
* @return Double the total about of cpu requirement for executor
*/
@@ -336,20 +339,20 @@ public class TopologyDetails {
/**
* Get the total CPU requirement for executor.
+ *
* @param exec
- * @return Map<String, Double> generic resource mapping requirement for the executor
+ * @return Map<String , Double> generic resource mapping requirement for the executor
*/
public Double getTotalCpuReqTask(ExecutorDetails exec) {
if (hasExecInTopo(exec)) {
return resourceList
- .get(exec).getTotalCpu();
+ .get(exec).getTotalCpu();
}
return null;
}
/**
- * Note: The public API relevant to resource aware scheduling is unstable as of May 2015.
- * We reserve the right to change them.
+ * Note: The public API relevant to resource aware scheduling is unstable as of May 2015. We reserve the right to change them.
*
* @return the total on-heap memory requested for this topology
*/
@@ -379,8 +382,7 @@ public class TopologyDetails {
}
/**
- * Note: The public API relevant to resource aware scheduling is unstable as of May 2015.
- * We reserve the right to change them.
+ * Note: The public API relevant to resource aware scheduling is unstable as of May 2015. We reserve the right to change them.
*
* @return the total off-heap memory requested for this topology
*/
@@ -410,8 +412,7 @@ public class TopologyDetails {
}
/**
- * Note: The public API relevant to resource aware scheduling is unstable as of May 2015.
- * We reserve the right to change them.
+ * Note: The public API relevant to resource aware scheduling is unstable as of May 2015. We reserve the right to change them.
*
* @return the total cpu requested for this topology
*/
@@ -428,6 +429,7 @@ public class TopologyDetails {
/**
* get the resources requirements for a executor.
+ *
* @param exec
* @return a map containing the resource requirements for this exec
*/
@@ -440,6 +442,7 @@ public class TopologyDetails {
/**
* Checks if a executor is part of this topology.
+ *
* @return Boolean whether or not a certain ExecutorDetail is included in the resourceList.
*/
public boolean hasExecInTopo(ExecutorDetails exec) {
@@ -472,28 +475,30 @@ public class TopologyDetails {
if (component.equals(Acker.ACKER_COMPONENT_ID)) {
if (topologyConf.containsKey(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB)) {
resourceListForExec.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB,
- ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB)));
+ ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB)));
}
if (topologyConf.containsKey(Config.TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB)) {
resourceListForExec.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB,
- ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB)));
+ ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB)));
}
if (topologyConf.containsKey(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT)) {
resourceListForExec.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT,
- ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT)));
+ ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT)));
}
} else if (component.startsWith(Constants.METRICS_COMPONENT_ID_PREFIX)) {
if (topologyConf.containsKey(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_ONHEAP_MEMORY_MB)) {
resourceListForExec.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB,
- ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_ONHEAP_MEMORY_MB)));
+ ObjectReader
+ .getDouble(topologyConf.get(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_ONHEAP_MEMORY_MB)));
}
if (topologyConf.containsKey(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_OFFHEAP_MEMORY_MB)) {
resourceListForExec.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB,
- ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_OFFHEAP_MEMORY_MB)));
+ ObjectReader
+ .getDouble(topologyConf.get(Config.TOPOLOGY_METRICS_CONSUMER_RESOURCES_OFFHEAP_MEMORY_MB)));
}
if (topologyConf.containsKey(Config.TOPOLOGY_METRICS_CONSUMER_CPU_PCORE_PERCENT)) {
resourceListForExec.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT,
- ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_METRICS_CONSUMER_CPU_PCORE_PERCENT)));
+ ObjectReader.getDouble(topologyConf.get(Config.TOPOLOGY_METRICS_CONSUMER_CPU_PCORE_PERCENT)));
}
}
}
@@ -514,6 +519,7 @@ public class TopologyDetails {
/**
* Get the max heap size for a worker used by this topology.
+ *
* @return the worker max heap size
*/
public Double getTopologyWorkerMaxHeapSize() {
@@ -551,17 +557,17 @@ public class TopologyDetails {
@Override
public String toString() {
return "Name: "
- + getName()
- + " id: "
- + getId()
- + " Priority: "
- + getTopologyPriority()
- + " Uptime: "
- + getUpTime()
- + " CPU: "
- + getTotalRequestedCpu()
- + " Memory: "
- + (getTotalRequestedMemOffHeap() + getTotalRequestedMemOnHeap());
+ + getName()
+ + " id: "
+ + getId()
+ + " Priority: "
+ + getTopologyPriority()
+ + " Uptime: "
+ + getUpTime()
+ + " CPU: "
+ + getTotalRequestedCpu()
+ + " Memory: "
+ + (getTotalRequestedMemOffHeap() + getTotalRequestedMemOnHeap());
}
@Override
[32/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-cassandra
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/ObjectMapperCqlStatementMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/ObjectMapperCqlStatementMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/ObjectMapperCqlStatementMapper.java
index fa9a348..4f5a021 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/ObjectMapperCqlStatementMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/ObjectMapperCqlStatementMapper.java
@@ -1,20 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.cassandra.query.impl;
@@ -54,7 +47,8 @@ public class ObjectMapperCqlStatementMapper implements CQLStatementTupleMapper {
private final Collection<TypeCodec<?>> codecs;
private final Collection<Class<?>> udtClasses;
- public ObjectMapperCqlStatementMapper(String operationField, String valueField, String timestampField, String ttlField, String consistencyLevelField, Collection<TypeCodec<?>> codecs, Collection<Class<?>> udtClasses) {
+ public ObjectMapperCqlStatementMapper(String operationField, String valueField, String timestampField, String ttlField,
+ String consistencyLevelField, Collection<TypeCodec<?>> codecs, Collection<Class<?>> udtClasses) {
Preconditions.checkNotNull(operationField, "Operation field must not be null");
Preconditions.checkNotNull(valueField, "Value field should not be null");
this.operationField = operationField;
@@ -68,14 +62,15 @@ public class ObjectMapperCqlStatementMapper implements CQLStatementTupleMapper {
@Override
public List<Statement> map(Map<String, Object> map, Session session, ITuple tuple) {
- final ObjectMapperOperation operation = (ObjectMapperOperation)tuple.getValueByField(operationField);
+ final ObjectMapperOperation operation = (ObjectMapperOperation) tuple.getValueByField(operationField);
Preconditions.checkNotNull(operation, "Operation must not be null");
final Object value = tuple.getValueByField(valueField);
final Object timestampObject = timestampField != null ? tuple.getValueByField(timestampField) : null;
final Object ttlObject = ttlField != null ? tuple.getValueByField(ttlField) : null;
- final ConsistencyLevel consistencyLevel = consistencyLevelField != null ? (ConsistencyLevel) tuple.getValueByField(consistencyLevelField) : null;
+ final ConsistencyLevel consistencyLevel =
+ consistencyLevelField != null ? (ConsistencyLevel) tuple.getValueByField(consistencyLevelField) : null;
final Class<?> valueClass = value.getClass();
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/PreparedStatementBinder.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/PreparedStatementBinder.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/PreparedStatementBinder.java
index 677ef56..5a4d39e 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/PreparedStatementBinder.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/PreparedStatementBinder.java
@@ -1,41 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query.impl;
import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.CodecRegistry;
import com.datastax.driver.core.PreparedStatement;
-import com.datastax.driver.core.TupleValue;
-import com.datastax.driver.core.TypeTokens;
-import com.datastax.driver.core.UDTValue;
-import org.apache.storm.cassandra.query.Column;
-
import java.io.Serializable;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.Date;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
+import org.apache.storm.cassandra.query.Column;
/**
*
@@ -66,15 +48,15 @@ public interface PreparedStatementBinder extends Serializable {
Object[] values = Column.getVals(columns);
BoundStatement boundStatement = statement.bind();
- for(Column col : columns) {
+ for (Column col : columns) {
// For native protocol V3 or below, all variables must be bound.
// With native protocol V4 or above, variables can be left unset,
// in which case they will be ignored server side (no tombstones will be generated).
- if(col.isNull()) {
+ if (col.isNull()) {
boundStatement.setToNull(col.getColumnName());
} else {
boundStatement.set(col.getColumnName(), col.getVal(),
- CodecRegistry.DEFAULT_INSTANCE.codecFor(col.getVal()));
+ CodecRegistry.DEFAULT_INSTANCE.codecFor(col.getVal()));
}
}
return statement.bind(values);
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/RoutingKeyGenerator.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/RoutingKeyGenerator.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/RoutingKeyGenerator.java
index 9c84506..948a21f 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/RoutingKeyGenerator.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/RoutingKeyGenerator.java
@@ -1,32 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query.impl;
import com.datastax.driver.core.CodecRegistry;
import com.datastax.driver.core.ProtocolVersion;
import com.google.common.base.Preconditions;
-import org.apache.storm.tuple.ITuple;
-
import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import org.apache.storm.tuple.ITuple;
public class RoutingKeyGenerator implements Serializable {
@@ -43,7 +36,7 @@ public class RoutingKeyGenerator implements Serializable {
public List<ByteBuffer> getRoutingKeys(ITuple tuple) {
List<ByteBuffer> keys = new ArrayList<>(routingKeys.size());
- for(String s : routingKeys) {
+ for (String s : routingKeys) {
Object value = tuple.getValueByField(s);
ByteBuffer serialized = CodecRegistry.DEFAULT_INSTANCE.codecFor(value).serialize(value, ProtocolVersion.NEWEST_SUPPORTED);
keys.add(serialized);
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/SimpleCQLStatementMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/SimpleCQLStatementMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/SimpleCQLStatementMapper.java
index 532bf6d..17a9049 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/SimpleCQLStatementMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/SimpleCQLStatementMapper.java
@@ -1,36 +1,29 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query.impl;
-import org.apache.storm.tuple.ITuple;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.SimpleStatement;
import com.datastax.driver.core.Statement;
import com.google.common.base.Preconditions;
-import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
-import org.apache.storm.cassandra.query.Column;
-import org.apache.storm.cassandra.query.CqlMapper;
-
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
+import org.apache.storm.cassandra.query.Column;
+import org.apache.storm.cassandra.query.CqlMapper;
+import org.apache.storm.tuple.ITuple;
/**
*
@@ -71,12 +64,13 @@ public class SimpleCQLStatementMapper implements CQLStatementTupleMapper {
List<Column> columns = mapper.map(tuple);
SimpleStatement statement = new SimpleStatement(queryString, Column.getVals(columns));
- if(hasRoutingKeys()) {
+ if (hasRoutingKeys()) {
List<ByteBuffer> keys = rkGenerator.getRoutingKeys(tuple);
- if( keys.size() == 1)
+ if (keys.size() == 1) {
statement.setRoutingKey(keys.get(0));
- else
+ } else {
statement.setRoutingKey(keys.toArray(new ByteBuffer[keys.size()]));
+ }
}
return Arrays.asList((Statement) statement);
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/selector/FieldSelector.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/selector/FieldSelector.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/selector/FieldSelector.java
index 3349bc3..dd9222a 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/selector/FieldSelector.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/selector/FieldSelector.java
@@ -1,28 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query.selector;
-import org.apache.storm.tuple.ITuple;
import com.datastax.driver.core.utils.UUIDs;
-import org.apache.storm.cassandra.query.Column;
-
import java.io.Serializable;
+import org.apache.storm.cassandra.query.Column;
+import org.apache.storm.tuple.ITuple;
public class FieldSelector implements Serializable {
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraBackingMap.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraBackingMap.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraBackingMap.java
index 029710e..61f9001 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraBackingMap.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraBackingMap.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
import com.datastax.driver.core.HostDistance;
@@ -23,6 +17,11 @@ import com.datastax.driver.core.PoolingOptions;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
import com.google.common.base.Preconditions;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Semaphore;
import org.apache.storm.cassandra.client.SimpleClient;
import org.apache.storm.cassandra.client.SimpleClientProvider;
import org.apache.storm.cassandra.query.AyncCQLResultSetValuesMapper;
@@ -41,12 +40,6 @@ import org.apache.storm.tuple.Values;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Semaphore;
-
/**
* An IBackingState implementation for Cassandra.
*
@@ -99,8 +92,8 @@ public class CassandraBackingMap<T> implements IBackingMap<T> {
if (options.maxParallelism == null || options.maxParallelism <= 0) {
PoolingOptions po = session.getCluster().getConfiguration().getPoolingOptions();
Integer maxRequestsPerHost = Math.min(
- po.getMaxConnectionsPerHost(HostDistance.LOCAL) * po.getMaxRequestsPerConnection(HostDistance.LOCAL),
- po.getMaxConnectionsPerHost(HostDistance.REMOTE) * po.getMaxRequestsPerConnection(HostDistance.REMOTE)
+ po.getMaxConnectionsPerHost(HostDistance.LOCAL) * po.getMaxRequestsPerConnection(HostDistance.LOCAL),
+ po.getMaxConnectionsPerHost(HostDistance.REMOTE) * po.getMaxRequestsPerConnection(HostDistance.REMOTE)
);
options.maxParallelism = maxRequestsPerHost / 2;
LOG.info("Parallelism default set to {}", options.maxParallelism);
@@ -127,7 +120,7 @@ public class CassandraBackingMap<T> implements IBackingMap<T> {
}
List<List<Values>> results = getResultMapper
- .map(session, selects, keyTuples);
+ .map(session, selects, keyTuples);
List<T> states = new ArrayList<>();
for (List<Values> values : results) {
@@ -228,12 +221,12 @@ public class CassandraBackingMap<T> implements IBackingMap<T> {
@Override
public String toString() {
return String.format("%s: [keys: %s, StateMapper: %s, getMapper: %s, putMapper: %s, maxParallelism: %d",
- this.getClass().getSimpleName(),
- keyFields,
- stateMapper,
- getMapper,
- putMapper,
- maxParallelism
+ this.getClass().getSimpleName(),
+ keyFields,
+ stateMapper,
+ getMapper,
+ putMapper,
+ maxParallelism
);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraMapStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraMapStateFactory.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraMapStateFactory.java
index 75f0d4f..5b82a6b 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraMapStateFactory.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraMapStateFactory.java
@@ -1,23 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
+import java.util.Map;
import org.apache.storm.task.IMetricsContext;
import org.apache.storm.trident.state.OpaqueValue;
import org.apache.storm.trident.state.State;
@@ -31,8 +26,6 @@ import org.apache.storm.trident.state.map.NonTransactionalMap;
import org.apache.storm.trident.state.map.OpaqueMap;
import org.apache.storm.trident.state.map.TransactionalMap;
-import java.util.Map;
-
/**
* A StateFactory implementation that creates a MapState backed by CassandraBackingMap.
*
@@ -78,8 +71,8 @@ public class CassandraMapStateFactory implements StateFactory {
cassandraBackingMap.prepare();
IBackingMap backingMap = cacheSize > 0
- ? new CachedMap<>(cassandraBackingMap, cacheSize)
- : cassandraBackingMap;
+ ? new CachedMap<>(cassandraBackingMap, cacheSize)
+ : cassandraBackingMap;
MapState<?> mapState;
@@ -89,7 +82,7 @@ public class CassandraMapStateFactory implements StateFactory {
break;
case TRANSACTIONAL:
- mapState = TransactionalMap.build((IBackingMap<TransactionalValue>)backingMap);
+ mapState = TransactionalMap.build((IBackingMap<TransactionalValue>) backingMap);
break;
case NON_TRANSACTIONAL:
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraQuery.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraQuery.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraQuery.java
index ca4416d..4fafa61 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraQuery.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraQuery.java
@@ -1,29 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
-import org.apache.storm.tuple.Values;
+import java.util.List;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseQueryFunction;
import org.apache.storm.trident.tuple.TridentTuple;
-
-import java.util.List;
+import org.apache.storm.tuple.Values;
/**
*
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraState.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraState.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraState.java
index 7fa138f..9ecae36 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraState.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraState.java
@@ -1,43 +1,36 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
-import org.apache.storm.topology.FailedException;
-import org.apache.storm.tuple.Values;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
import com.google.common.base.Preconditions;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.cassandra.client.SimpleClient;
import org.apache.storm.cassandra.client.SimpleClientProvider;
import org.apache.storm.cassandra.query.CQLResultSetValuesMapper;
import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.storm.topology.FailedException;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.tuple.TridentTuple;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import org.apache.storm.tuple.Values;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
@@ -57,33 +50,6 @@ public class CassandraState implements State {
this.options = options;
}
- public static final class Options implements Serializable {
- private final SimpleClientProvider clientProvider;
- private CQLStatementTupleMapper cqlStatementTupleMapper;
- private CQLResultSetValuesMapper cqlResultSetValuesMapper;
- private BatchStatement.Type batchingType;
-
- public Options(SimpleClientProvider clientProvider) {
- this.clientProvider = clientProvider;
- }
-
- public Options withCQLStatementTupleMapper(CQLStatementTupleMapper cqlStatementTupleMapper) {
- this.cqlStatementTupleMapper = cqlStatementTupleMapper;
- return this;
- }
-
- public Options withCQLResultSetValuesMapper(CQLResultSetValuesMapper cqlResultSetValuesMapper) {
- this.cqlResultSetValuesMapper = cqlResultSetValuesMapper;
- return this;
- }
-
- public Options withBatching(BatchStatement.Type batchingType) {
- this.batchingType = batchingType;
- return this;
- }
-
- }
-
@Override
public void beginCommit(Long txid) {
LOG.debug("beginCommit is no operation");
@@ -155,4 +121,31 @@ public class CassandraState implements State {
return batchRetrieveResult;
}
+ public static final class Options implements Serializable {
+ private final SimpleClientProvider clientProvider;
+ private CQLStatementTupleMapper cqlStatementTupleMapper;
+ private CQLResultSetValuesMapper cqlResultSetValuesMapper;
+ private BatchStatement.Type batchingType;
+
+ public Options(SimpleClientProvider clientProvider) {
+ this.clientProvider = clientProvider;
+ }
+
+ public Options withCQLStatementTupleMapper(CQLStatementTupleMapper cqlStatementTupleMapper) {
+ this.cqlStatementTupleMapper = cqlStatementTupleMapper;
+ return this;
+ }
+
+ public Options withCQLResultSetValuesMapper(CQLResultSetValuesMapper cqlResultSetValuesMapper) {
+ this.cqlResultSetValuesMapper = cqlResultSetValuesMapper;
+ return this;
+ }
+
+ public Options withBatching(BatchStatement.Type batchingType) {
+ this.batchingType = batchingType;
+ return this;
+ }
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraStateFactory.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraStateFactory.java
index a7f9e9c..fcdbb1c 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraStateFactory.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraStateFactory.java
@@ -1,32 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
-import org.apache.storm.task.IMetricsContext;
+import java.util.Map;
import org.apache.storm.cassandra.CassandraContext;
import org.apache.storm.cassandra.query.CQLResultSetValuesMapper;
import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
+import org.apache.storm.task.IMetricsContext;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.state.StateFactory;
-import java.util.Map;
-
/**
*
*/
@@ -38,7 +31,8 @@ public class CassandraStateFactory implements StateFactory {
}
public CassandraStateFactory(CQLStatementTupleMapper cqlStatementTupleMapper, CQLResultSetValuesMapper cqlResultSetValuesMapper) {
- this(new CassandraState.Options(new CassandraContext()).withCQLStatementTupleMapper(cqlStatementTupleMapper).withCQLResultSetValuesMapper(cqlResultSetValuesMapper));
+ this(new CassandraState.Options(new CassandraContext()).withCQLStatementTupleMapper(cqlStatementTupleMapper)
+ .withCQLResultSetValuesMapper(cqlResultSetValuesMapper));
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraStateUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraStateUpdater.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraStateUpdater.java
index ad185b9..e4bb380 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraStateUpdater.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/CassandraStateUpdater.java
@@ -1,29 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
+import java.util.List;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseStateUpdater;
import org.apache.storm.trident.tuple.TridentTuple;
-import java.util.List;
-
/**
*
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/MapStateFactoryBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/MapStateFactoryBuilder.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/MapStateFactoryBuilder.java
index e36b7ef..ed8ab71 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/MapStateFactoryBuilder.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/MapStateFactoryBuilder.java
@@ -1,25 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
import com.datastax.driver.core.querybuilder.Insert;
import com.datastax.driver.core.querybuilder.Select;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
import org.apache.storm.cassandra.CassandraContext;
import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
import org.apache.storm.trident.state.JSONNonTransactionalSerializer;
@@ -34,12 +33,10 @@ import org.apache.storm.tuple.Fields;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.*;
+import static com.datastax.driver.core.querybuilder.QueryBuilder.bindMarker;
+import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
+import static com.datastax.driver.core.querybuilder.QueryBuilder.insertInto;
+import static com.datastax.driver.core.querybuilder.QueryBuilder.select;
import static org.apache.storm.cassandra.DynamicStatementBuilder.all;
import static org.apache.storm.cassandra.DynamicStatementBuilder.boundQuery;
@@ -85,20 +82,20 @@ public class MapStateFactoryBuilder<T> {
public static <U> MapStateFactoryBuilder<OpaqueValue<U>> opaque(Map<String, Object> cassandraConf) {
return new MapStateFactoryBuilder<OpaqueValue<U>>()
- .withStateType(StateType.OPAQUE)
- .withCassandraConfig(cassandraConf);
+ .withStateType(StateType.OPAQUE)
+ .withCassandraConfig(cassandraConf);
}
public static <U> MapStateFactoryBuilder<TransactionalValue<U>> transactional(Map<String, Object> cassandraConf) {
return new MapStateFactoryBuilder<TransactionalValue<U>>()
- .withStateType(StateType.TRANSACTIONAL)
- .withCassandraConfig(cassandraConf);
+ .withStateType(StateType.TRANSACTIONAL)
+ .withCassandraConfig(cassandraConf);
}
public static <U> MapStateFactoryBuilder<U> nontransactional(Map<String, Object> cassandraConf) {
return new MapStateFactoryBuilder<U>()
- .withStateType(StateType.NON_TRANSACTIONAL)
- .withCassandraConfig(cassandraConf);
+ .withStateType(StateType.NON_TRANSACTIONAL)
+ .withCassandraConfig(cassandraConf);
}
public MapStateFactoryBuilder<T> withTable(String keyspace, String table) {
@@ -167,7 +164,7 @@ public class MapStateFactoryBuilder<T> {
Objects.requireNonNull(stateType, "A state type must be specified.");
List<String> stateFields = stateMapper.getStateFields()
- .toList();
+ .toList();
String[] stateFieldsArray = stateFields.toArray(new String[stateFields.size()]);
@@ -177,47 +174,47 @@ public class MapStateFactoryBuilder<T> {
// Build get query
Select.Where getQuery = select(stateFieldsArray)
- .from(keyspace, table)
- .where();
+ .from(keyspace, table)
+ .where();
for (String key : keys) {
getQuery.and(eq(key, bindMarker()));
}
CQLStatementTupleMapper get = boundQuery(getQuery.toString())
- .bind(all())
- .build();
+ .bind(all())
+ .build();
// Build put query
Insert putStatement = insertInto(keyspace, table)
- .values(allFields, Collections.<Object>nCopies(allFields.size(), bindMarker()));
+ .values(allFields, Collections.<Object>nCopies(allFields.size(), bindMarker()));
CQLStatementTupleMapper put = boundQuery(putStatement.toString())
- .bind(all())
- .build();
+ .bind(all())
+ .build();
CassandraBackingMap.Options options = new CassandraBackingMap.Options<T>(new CassandraContext())
- .withGetMapper(get)
- .withPutMapper(put)
- .withStateMapper(stateMapper)
- .withKeys(new Fields(keys))
- .withMaxParallelism(maxParallelism);
+ .withGetMapper(get)
+ .withPutMapper(put)
+ .withStateMapper(stateMapper)
+ .withKeys(new Fields(keys))
+ .withMaxParallelism(maxParallelism);
logger.debug("Building factory with: \n get: {}\n put: {}\n mapper: {}",
- getQuery.toString(),
- putStatement.toString(),
- stateMapper.toString());
+ getQuery.toString(),
+ putStatement.toString(),
+ stateMapper.toString());
switch (stateType) {
case NON_TRANSACTIONAL:
return CassandraMapStateFactory.nonTransactional(options, cassandraConfig)
- .withCache(cacheSize);
+ .withCache(cacheSize);
case TRANSACTIONAL:
return CassandraMapStateFactory.transactional(options, cassandraConfig)
- .withCache(cacheSize);
+ .withCache(cacheSize);
case OPAQUE:
return CassandraMapStateFactory.opaque(options, cassandraConfig)
- .withCache(cacheSize);
+ .withCache(cacheSize);
}
return null;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/NonTransactionalTupleStateMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/NonTransactionalTupleStateMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/NonTransactionalTupleStateMapper.java
index 3a36b07..cdaa2a1 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/NonTransactionalTupleStateMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/NonTransactionalTupleStateMapper.java
@@ -1,29 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
+import java.util.List;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.ITuple;
import org.apache.storm.tuple.Values;
-import java.util.List;
-
/**
* State mapper that maps a tuple to separate state fields.
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/OpaqueTupleStateMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/OpaqueTupleStateMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/OpaqueTupleStateMapper.java
index 882c9b1..a7e857b 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/OpaqueTupleStateMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/OpaqueTupleStateMapper.java
@@ -1,31 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.storm.trident.state.OpaqueValue;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.ITuple;
import org.apache.storm.tuple.Values;
-import java.util.ArrayList;
-import java.util.List;
-
/**
* State mapper that maps an opaque tuple to separate state fields.
*/
@@ -64,8 +57,7 @@ public class OpaqueTupleStateMapper implements StateMapper<OpaqueValue<ITuple>>
for (String valueField : tupleFields) {
if (tuple.getCurr() != null) {
values.add(tuple.getCurr().getValueByField(valueField));
- }
- else {
+ } else {
values.add(null);
}
}
@@ -73,8 +65,7 @@ public class OpaqueTupleStateMapper implements StateMapper<OpaqueValue<ITuple>>
for (String valueField : tupleFields) {
if (tuple.getPrev() != null) {
values.add(tuple.getPrev().getValueByField(valueField));
- }
- else {
+ } else {
values.add(null);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SerializedStateMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SerializedStateMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SerializedStateMapper.java
index b4ec6c8..49b43fd 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SerializedStateMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SerializedStateMapper.java
@@ -1,30 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
+import java.nio.ByteBuffer;
+import java.util.List;
import org.apache.storm.trident.state.Serializer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
-import java.nio.ByteBuffer;
-import java.util.List;
-
public class SerializedStateMapper<T> implements StateMapper<T> {
private final Fields stateFields;
@@ -50,12 +43,10 @@ public class SerializedStateMapper<T> implements StateMapper<T> {
public T fromValues(List<Values> values) {
if (values.size() == 0) {
return null;
- }
- else if (values.size() == 1) {
+ } else if (values.size() == 1) {
ByteBuffer bytes = (ByteBuffer) values.get(0).get(0);
return serializer.deserialize(bytes.array());
- }
- else {
+ } else {
throw new IllegalArgumentException("Can only convert single values, " + values.size() + " encountered");
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SimpleStateMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SimpleStateMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SimpleStateMapper.java
index cc03a09..0bc3191 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SimpleStateMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SimpleStateMapper.java
@@ -1,31 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
+import java.util.List;
import org.apache.storm.trident.state.OpaqueValue;
import org.apache.storm.trident.state.StateType;
import org.apache.storm.trident.state.TransactionalValue;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
-import java.util.List;
-
public class SimpleStateMapper<T> implements StateMapper<T> {
private final Fields fields;
@@ -77,8 +70,7 @@ public class SimpleStateMapper<T> implements StateMapper<T> {
public T fromValues(List<Values> valuesSet) {
if (valuesSet == null || valuesSet.size() == 0) {
return null;
- }
- else if (valuesSet.size() == 1) {
+ } else if (valuesSet.size() == 1) {
Values values = valuesSet.get(0);
if (values == null) {
return null;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SimpleTuple.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SimpleTuple.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SimpleTuple.java
index 6424bc0..7bc066e 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SimpleTuple.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/SimpleTuple.java
@@ -1,30 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.cassandra.trident.state;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.ITuple;
+package org.apache.storm.cassandra.trident.state;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.ITuple;
/**
* Utility class for passing around ordered key/value data with an immutable key set.
@@ -60,18 +53,12 @@ public class SimpleTuple implements ITuple, Serializable {
int index = keys.indexOf(key);
if (index >= 0) {
values.set(index, value);
- }
- else {
+ } else {
throw new IllegalArgumentException("Field " + key + " does not exist.");
}
return this;
}
- public SimpleTuple setValues(List<Object> values) {
- this.values = new ArrayList<>(values);
- return this;
- }
-
@Override
public int size() {
return keys.size();
@@ -206,6 +193,11 @@ public class SimpleTuple implements ITuple, Serializable {
return Collections.unmodifiableList(values);
}
+ public SimpleTuple setValues(List<Object> values) {
+ this.values = new ArrayList<>(values);
+ return this;
+ }
+
public List<String> getKeys() {
return Collections.unmodifiableList(keys);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/StateMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/StateMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/StateMapper.java
index ef0c783..257c3ba 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/StateMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/StateMapper.java
@@ -1,28 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.cassandra.trident.state;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Values;
+package org.apache.storm.cassandra.trident.state;
import java.io.Serializable;
import java.util.List;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Values;
public interface StateMapper<T> extends Serializable {
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TransactionalTupleStateMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TransactionalTupleStateMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TransactionalTupleStateMapper.java
index 83332b9..7b3d670 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TransactionalTupleStateMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TransactionalTupleStateMapper.java
@@ -1,31 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.storm.trident.state.TransactionalValue;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.ITuple;
import org.apache.storm.tuple.Values;
-import java.util.ArrayList;
-import java.util.List;
-
/**
* State mapper that maps a transactional tuple to separate state fields.
*/
@@ -61,8 +54,7 @@ public class TransactionalTupleStateMapper implements StateMapper<TransactionalV
for (String valueField : tupleFields) {
if (tuple.getVal() != null) {
values.add(tuple.getVal().getValueByField(valueField));
- }
- else {
+ } else {
values.add(null);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TridentAyncCQLResultSetValuesMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TridentAyncCQLResultSetValuesMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TridentAyncCQLResultSetValuesMapper.java
index 0d03fed..0159eef 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TridentAyncCQLResultSetValuesMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TridentAyncCQLResultSetValuesMapper.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
import com.datastax.driver.core.ResultSet;
@@ -23,6 +17,9 @@ import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
import com.google.common.util.concurrent.SettableFuture;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Semaphore;
import org.apache.storm.cassandra.executor.AsyncExecutor;
import org.apache.storm.cassandra.executor.AsyncExecutorProvider;
import org.apache.storm.cassandra.executor.AsyncResultHandler;
@@ -33,10 +30,6 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.ITuple;
import org.apache.storm.tuple.Values;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Semaphore;
-
/**
* A result set mapper implementation which runs requests in parallel and waits for them all to finish.
*/
@@ -113,5 +106,4 @@ public class TridentAyncCQLResultSetValuesMapper implements AyncCQLResultSetValu
}
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TridentResultSetValuesMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TridentResultSetValuesMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TridentResultSetValuesMapper.java
index 69cd6f8..cd34362 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TridentResultSetValuesMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/trident/state/TridentResultSetValuesMapper.java
@@ -1,35 +1,28 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident.state;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.ITuple;
-import org.apache.storm.tuple.Values;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
-import org.apache.storm.cassandra.query.CQLResultSetValuesMapper;
-
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import org.apache.storm.cassandra.query.CQLResultSetValuesMapper;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.ITuple;
+import org.apache.storm.tuple.Values;
/**
*
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/WeatherSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/WeatherSpout.java b/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/WeatherSpout.java
index caf19ee..1043130 100644
--- a/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/WeatherSpout.java
+++ b/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/WeatherSpout.java
@@ -1,23 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -26,9 +22,6 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.junit.Assert;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
public class WeatherSpout extends BaseRichSpout {
private SpoutOutputCollector spoutOutputCollector;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/testtools/EmbeddedCassandraResource.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/testtools/EmbeddedCassandraResource.java b/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/testtools/EmbeddedCassandraResource.java
index 0f02bb4..57dd6b9 100644
--- a/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/testtools/EmbeddedCassandraResource.java
+++ b/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/testtools/EmbeddedCassandraResource.java
@@ -1,22 +1,15 @@
/**
* Copyright (c) 2009-2011 VMware, Inc. All Rights Reserved.
*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.cassandra.testtools;
@@ -25,12 +18,10 @@ import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
-
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.service.CassandraDaemon;
import org.apache.cassandra.utils.FBUtilities;
-
import org.junit.rules.ExternalResource;
/**
@@ -41,7 +32,8 @@ import org.junit.rules.ExternalResource;
* This implementation is based on the springsource community project,
* com.springsource.insight:insight-plugin-cassandra12.
*
- * {@see <a href="https://github.com/spring-projects/spring-insight-plugins/blob/c2986b457b482cd08a77a26297c087df59535067/collection-plugins/cassandra12/src/test/java/com/springsource/insight/plugin/cassandra/embeded/EmbeddedCassandraService.java">
+ * {@see <a href="https://github.com/spring-projects/spring-insight-plugins/blob/c2986b457b482cd08a77a26297c087df59535067/collection
+ * -plugins/cassandra12/src/test/java/com/springsource/insight/plugin/cassandra/embeded/EmbeddedCassandraService.java">
* com.springsource.insight:insight-plugin-cassandra12
* </a>}
*
@@ -52,10 +44,9 @@ import org.junit.rules.ExternalResource;
*/
public class EmbeddedCassandraResource extends ExternalResource {
- CassandraDaemon cassandraDaemon;
-
private final String host;
private final Integer nativeTransportPort;
+ CassandraDaemon cassandraDaemon;
public EmbeddedCassandraResource() {
try {
@@ -64,8 +55,7 @@ public class EmbeddedCassandraResource extends ExternalResource {
cassandraDaemon.init(null);
host = DatabaseDescriptor.getRpcAddress().getHostName();
nativeTransportPort = DatabaseDescriptor.getNativeTransportPort();
- }
- catch (Exception e) {
+ } catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
}
@@ -83,8 +73,7 @@ public class EmbeddedCassandraResource extends ExternalResource {
if (FBUtilities.isWindows()) {
cassandraDaemon.thriftServer.stop();
cassandraDaemon.nativeServer.stop();
- }
- else {
+ } else {
cassandraDaemon.stop();
}
[30/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-hdfs
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-hdfs
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7da98cf0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7da98cf0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7da98cf0
Branch: refs/heads/master
Commit: 7da98cf0c5d3e23fac42871974ff8017924673c5
Parents: 1872317
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 21:45:33 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 00:22:35 2018 -0400
----------------------------------------------------------------------
external/storm-hdfs/pom.xml | 2 +-
.../storm/hdfs/avro/AbstractAvroSerializer.java | 33 +-
.../storm/hdfs/avro/AvroSchemaRegistry.java | 22 +-
.../org/apache/storm/hdfs/avro/AvroUtils.java | 24 +-
.../hdfs/avro/ConfluentAvroSerializer.java | 24 +-
.../storm/hdfs/avro/FixedAvroSerializer.java | 30 +-
.../storm/hdfs/avro/GenericAvroSerializer.java | 19 +-
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 79 ++-
.../storm/hdfs/bolt/AvroGenericRecordBolt.java | 52 +-
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 51 +-
.../storm/hdfs/bolt/SequenceFileBolt.java | 50 +-
.../java/org/apache/storm/hdfs/bolt/Writer.java | 18 +-
.../hdfs/bolt/format/DefaultFileNameFormat.java | 32 +-
.../hdfs/bolt/format/DefaultSequenceFormat.java | 12 +-
.../hdfs/bolt/format/DelimitedRecordFormat.java | 29 +-
.../storm/hdfs/bolt/format/FileNameFormat.java | 22 +-
.../storm/hdfs/bolt/format/RecordFormat.java | 22 +-
.../storm/hdfs/bolt/format/SequenceFormat.java | 7 +-
.../hdfs/bolt/format/SimpleFileNameFormat.java | 38 +-
.../hdfs/bolt/rotation/FileRotationPolicy.java | 22 +-
.../bolt/rotation/FileSizeRotationPolicy.java | 62 +-
.../hdfs/bolt/rotation/NoRotationPolicy.java | 19 +-
.../hdfs/bolt/rotation/TimedRotationPolicy.java | 43 +-
.../storm/hdfs/bolt/sync/CountSyncPolicy.java | 21 +-
.../apache/storm/hdfs/bolt/sync/SyncPolicy.java | 22 +-
.../storm/hdfs/common/AbstractHDFSWriter.java | 22 +-
.../common/AvroGenericRecordHDFSWriter.java | 24 +-
.../apache/storm/hdfs/common/HDFSWriter.java | 26 +-
.../org/apache/storm/hdfs/common/HdfsUtils.java | 137 +++--
.../storm/hdfs/common/ModifTimeComparator.java | 27 +-
.../storm/hdfs/common/NullPartitioner.java | 19 +-
.../apache/storm/hdfs/common/Partitioner.java | 24 +-
.../storm/hdfs/common/SequenceFileWriter.java | 24 +-
.../hdfs/common/rotation/MoveFileAction.java | 6 +-
.../hdfs/common/rotation/RotationAction.java | 6 +-
.../storm/hdfs/spout/AbstractFileReader.java | 70 ++-
.../org/apache/storm/hdfs/spout/Configs.java | 45 +-
.../org/apache/storm/hdfs/spout/DirLock.java | 182 +++---
.../org/apache/storm/hdfs/spout/FileLock.java | 563 ++++++++++---------
.../org/apache/storm/hdfs/spout/FileOffset.java | 25 +-
.../org/apache/storm/hdfs/spout/FileReader.java | 49 +-
.../org/apache/storm/hdfs/spout/HdfsSpout.java | 161 +++---
.../apache/storm/hdfs/spout/ParseException.java | 24 +-
.../storm/hdfs/spout/ProgressTracker.java | 86 ++-
.../storm/hdfs/spout/SequenceFileReader.java | 340 ++++++-----
.../apache/storm/hdfs/spout/TextFileReader.java | 304 +++++-----
.../apache/storm/hdfs/trident/HdfsState.java | 322 +++++------
.../storm/hdfs/trident/HdfsStateFactory.java | 12 +-
.../apache/storm/hdfs/trident/HdfsUpdater.java | 6 +-
.../trident/format/DefaultFileNameFormat.java | 29 +-
.../trident/format/DefaultSequenceFormat.java | 11 +-
.../trident/format/DelimitedRecordFormat.java | 31 +-
.../hdfs/trident/format/FileNameFormat.java | 19 +-
.../storm/hdfs/trident/format/RecordFormat.java | 22 +-
.../hdfs/trident/format/SequenceFormat.java | 7 +-
.../trident/format/SimpleFileNameFormat.java | 36 +-
.../trident/rotation/FileRotationPolicy.java | 24 +-
.../rotation/FileSizeRotationPolicy.java | 64 +--
.../hdfs/trident/rotation/NoRotationPolicy.java | 19 +-
.../trident/rotation/TimedRotationPolicy.java | 49 +-
.../hdfs/trident/sync/CountSyncPolicy.java | 21 +-
.../storm/hdfs/trident/sync/SyncPolicy.java | 22 +-
.../hdfs/avro/TestFixedAvroSerializer.java | 44 +-
.../hdfs/avro/TestGenericAvroSerializer.java | 33 +-
.../hdfs/bolt/AvroGenericRecordBoltTest.java | 129 ++---
.../apache/storm/hdfs/bolt/TestHdfsBolt.java | 135 ++---
.../storm/hdfs/bolt/TestSequenceFileBolt.java | 115 ++--
.../apache/storm/hdfs/bolt/TestWritersMap.java | 22 +-
.../bolt/format/TestSimpleFileNameFormat.java | 39 +-
.../apache/storm/hdfs/spout/ConfigsTest.java | 46 +-
.../apache/storm/hdfs/spout/TestDirLock.java | 29 +-
.../apache/storm/hdfs/spout/TestFileLock.java | 47 +-
.../storm/hdfs/spout/TestHdfsSemantics.java | 35 +-
.../apache/storm/hdfs/spout/TestHdfsSpout.java | 201 ++++---
.../storm/hdfs/spout/TestProgressTracker.java | 21 +-
.../storm/hdfs/testing/MiniDFSClusterRule.java | 8 +-
.../storm/hdfs/trident/HdfsStateTest.java | 103 ++--
.../format/TestSimpleFileNameFormat.java | 30 +-
78 files changed, 2128 insertions(+), 2522 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/pom.xml b/external/storm-hdfs/pom.xml
index 3fdc801..cb5baa2 100644
--- a/external/storm-hdfs/pom.xml
+++ b/external/storm-hdfs/pom.xml
@@ -267,7 +267,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>1406</maxAllowedViolations>
+ <maxAllowedViolations>189</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java
index ddf015d..adb842a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AbstractAvroSerializer.java
@@ -1,26 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.avro;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
+import java.io.IOException;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericContainer;
import org.apache.avro.generic.GenericDatumReader;
@@ -30,10 +26,9 @@ import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.EncoderFactory;
-import java.io.IOException;
-
//Generously adapted from:
-//https://github.com/kijiproject/kiji-express/blob/master/kiji-express/src/main/scala/org/kiji/express/flow/framework/serialization/AvroSerializer.scala
+//https://github.com/kijiproject/kiji-express/blob/master/kiji-express/src/main/scala/org/kiji/express/flow/framework/serialization
+// /AvroSerializer.scala
//Which has as an ASL2.0 license
/**
@@ -52,8 +47,8 @@ public abstract class AbstractAvroSerializer extends Serializer<GenericContainer
GenericDatumWriter<GenericContainer> writer = new GenericDatumWriter<>(record.getSchema());
BinaryEncoder encoder = EncoderFactory
- .get()
- .directBinaryEncoder(output, null);
+ .get()
+ .directBinaryEncoder(output, null);
try {
writer.write(record, encoder);
} catch (IOException e) {
@@ -66,8 +61,8 @@ public abstract class AbstractAvroSerializer extends Serializer<GenericContainer
Schema theSchema = this.getSchema(input.readString());
GenericDatumReader<GenericContainer> reader = new GenericDatumReader<>(theSchema);
Decoder decoder = DecoderFactory
- .get()
- .directBinaryDecoder(input, null);
+ .get()
+ .directBinaryDecoder(input, null);
GenericContainer foo;
try {
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java
index 0d1dc8b..cca5099 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroSchemaRegistry.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.avro;
-import org.apache.avro.Schema;
+package org.apache.storm.hdfs.avro;
import java.io.Serializable;
+import org.apache.avro.Schema;
public interface AvroSchemaRegistry extends Serializable {
String getFingerprint(Schema schema);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java
index 5549291..13798bb 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/AvroUtils.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.avro;
import org.apache.avro.generic.GenericData;
@@ -33,9 +28,8 @@ public class AvroUtils {
public static void addAvroKryoSerializations(Config conf) throws ClassNotFoundException {
final Class serializerClass;
if (conf.containsKey("avro.serializer")) {
- serializerClass = Class.forName((String)conf.get("avro.serializer"));
- }
- else {
+ serializerClass = Class.forName((String) conf.get("avro.serializer"));
+ } else {
serializerClass = GenericAvroSerializer.class;
}
conf.registerSerialization(GenericData.Record.class, serializerClass);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/ConfluentAvroSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/ConfluentAvroSerializer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/ConfluentAvroSerializer.java
index bb03a11..17f3eb7 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/ConfluentAvroSerializer.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/ConfluentAvroSerializer.java
@@ -1,30 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.avro;
import com.esotericsoftware.kryo.Kryo;
import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient;
import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient;
import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException;
-import org.apache.avro.Schema;
-
import java.io.IOException;
import java.util.Map;
+import org.apache.avro.Schema;
/**
* This class provides a mechanism to utilize the Confluent Schema Registry (https://github.com/confluentinc/schema-registry)
@@ -33,8 +27,8 @@ import java.util.Map;
*/
public class ConfluentAvroSerializer extends AbstractAvroSerializer {
- private SchemaRegistryClient theClient;
final private String url;
+ private SchemaRegistryClient theClient;
/**
* A constructor for use by test cases ONLY, thus the default scope.
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java
index 4dd5fdc..128e802 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/FixedAvroSerializer.java
@@ -1,25 +1,16 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.avro;
-import org.apache.avro.Schema;
-import org.apache.avro.SchemaNormalization;
-import org.apache.commons.codec.binary.Base64;
+package org.apache.storm.hdfs.avro;
import java.io.BufferedReader;
import java.io.IOException;
@@ -28,6 +19,9 @@ import java.io.InputStreamReader;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
import java.util.Map;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaNormalization;
+import org.apache.commons.codec.binary.Base64;
/**
* A class to help (de)serialize a pre-defined set of Avro schemas. Schemas should be listed, one per line, in a file
@@ -45,9 +39,9 @@ public class FixedAvroSerializer extends AbstractAvroSerializer {
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
String line;
- while((line = reader.readLine()) != null) {
+ while ((line = reader.readLine()) != null) {
Schema schema = new Schema.Parser().parse(line);
- byte [] fp = SchemaNormalization.parsingFingerprint(FP_ALGO, schema);
+ byte[] fp = SchemaNormalization.parsingFingerprint(FP_ALGO, schema);
String fingerPrint = new String(Base64.decodeBase64(fp));
fingerprint2schemaMap.put(fingerPrint, schema);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java
index fedf698..6bb0e26 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/avro/GenericAvroSerializer.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.avro;
import org.apache.avro.Schema;
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
index 10884a6..bcff104 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
@@ -1,28 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.topology.base.BaseRichBolt;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.utils.TupleUtils;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -34,19 +32,15 @@ import org.apache.storm.hdfs.common.NullPartitioner;
import org.apache.storm.hdfs.common.Partitioner;
import org.apache.storm.hdfs.common.rotation.RotationAction;
import org.apache.storm.hdfs.security.HdfsSecurityUtil;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseRichBolt;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.utils.TupleUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
-
public abstract class AbstractHdfsBolt extends BaseRichBolt {
private static final Logger LOG = LoggerFactory.getLogger(AbstractHdfsBolt.class);
private static final Integer DEFAULT_RETRY_COUNT = 3;
@@ -68,14 +62,13 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
protected String configKey;
protected transient Object writeLock;
protected transient Timer rotationTimer; // only used for TimedRotationPolicy
- private List<Tuple> tupleBatch = new LinkedList<>();
protected long offset = 0;
protected Integer fileRetryCount = DEFAULT_RETRY_COUNT;
protected Integer tickTupleInterval = DEFAULT_TICK_TUPLE_INTERVAL_SECS;
protected Integer maxOpenFiles = DEFAULT_MAX_OPEN_FILES;
protected Partitioner partitioner = new NullPartitioner();
-
protected transient Configuration hdfsConfig;
+ private List<Tuple> tupleBatch = new LinkedList<>();
protected void rotateOutputFile(Writer writer) throws IOException {
LOG.info("Rotating output file...");
@@ -98,7 +91,7 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
* @param topologyContext
* @param collector
*/
- public final void prepare(Map<String, Object> conf, TopologyContext topologyContext, OutputCollector collector){
+ public final void prepare(Map<String, Object> conf, TopologyContext topologyContext, OutputCollector collector) {
this.writeLock = new Object();
if (this.syncPolicy == null) throw new IllegalStateException("SyncPolicy must be specified.");
if (this.rotationPolicy == null) throw new IllegalStateException("RotationPolicy must be specified.");
@@ -111,21 +104,21 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
this.collector = collector;
this.fileNameFormat.prepare(conf, topologyContext);
this.hdfsConfig = new Configuration();
- Map<String, Object> map = (Map<String, Object>)conf.get(this.configKey);
- if(map != null){
- for(String key : map.keySet()){
+ Map<String, Object> map = (Map<String, Object>) conf.get(this.configKey);
+ if (map != null) {
+ for (String key : map.keySet()) {
this.hdfsConfig.set(key, String.valueOf(map.get(key)));
}
}
- try{
+ try {
HdfsSecurityUtil.login(conf, hdfsConfig);
doPrepare(conf, topologyContext, collector);
- } catch (Exception e){
+ } catch (Exception e) {
throw new RuntimeException("Error preparing HdfsBolt: " + e.getMessage(), e);
}
- if(this.rotationPolicy instanceof TimedRotationPolicy){
+ if (this.rotationPolicy instanceof TimedRotationPolicy) {
startTimedRotationPolicy();
}
}
@@ -280,7 +273,7 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
}
private void startTimedRotationPolicy() {
- long interval = ((TimedRotationPolicy)this.rotationPolicy).getInterval();
+ long interval = ((TimedRotationPolicy) this.rotationPolicy).getInterval();
this.rotationTimer = new Timer(true);
TimerTask task = new TimerTask() {
@Override
@@ -295,8 +288,7 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
final String partitionPath = this.partitioner.getPartitionPath(tuple);
final int rotation;
- if (rotationCounterMap.containsKey(partitionPath))
- {
+ if (rotationCounterMap.containsKey(partitionPath)) {
rotation = rotationCounterMap.get(partitionPath) + 1;
} else {
rotation = 0;
@@ -304,10 +296,11 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
rotationCounterMap.put(partitionPath, rotation);
return new Path(this.fsUrl + this.fileNameFormat.getPath() + partitionPath,
- this.fileNameFormat.getName(rotation, System.currentTimeMillis()));
+ this.fileNameFormat.getName(rotation, System.currentTimeMillis()));
}
- abstract protected void doPrepare(Map<String, Object> conf, TopologyContext topologyContext, OutputCollector collector) throws IOException;
+ abstract protected void doPrepare(Map<String, Object> conf, TopologyContext topologyContext, OutputCollector collector) throws
+ IOException;
abstract protected String getWriterKey(Tuple tuple);
@@ -318,7 +311,7 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
final OutputCollector collector;
public WritersMap(long maxWriters, OutputCollector collector) {
- super((int)maxWriters, 0.75f, true);
+ super((int) maxWriters, 0.75f, true);
this.maxWriters = maxWriters;
this.collector = collector;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java
index 9ab0e12..f00df3a 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBolt.java
@@ -1,73 +1,67 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt;
-import org.apache.storm.hdfs.common.AbstractHDFSWriter;
-import org.apache.storm.hdfs.common.AvroGenericRecordHDFSWriter;
-import org.apache.storm.hdfs.common.Partitioner;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Tuple;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Map;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
+import org.apache.storm.hdfs.common.AbstractHDFSWriter;
+import org.apache.storm.hdfs.common.AvroGenericRecordHDFSWriter;
+import org.apache.storm.hdfs.common.Partitioner;
import org.apache.storm.hdfs.common.rotation.RotationAction;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.net.URI;
-import java.util.Map;
-public class AvroGenericRecordBolt extends AbstractHdfsBolt{
+public class AvroGenericRecordBolt extends AbstractHdfsBolt {
private static final Logger LOG = LoggerFactory.getLogger(AvroGenericRecordBolt.class);
- public AvroGenericRecordBolt withFsUrl(String fsUrl){
+ public AvroGenericRecordBolt withFsUrl(String fsUrl) {
this.fsUrl = fsUrl;
return this;
}
- public AvroGenericRecordBolt withConfigKey(String configKey){
+ public AvroGenericRecordBolt withConfigKey(String configKey) {
this.configKey = configKey;
return this;
}
- public AvroGenericRecordBolt withFileNameFormat(FileNameFormat fileNameFormat){
+ public AvroGenericRecordBolt withFileNameFormat(FileNameFormat fileNameFormat) {
this.fileNameFormat = fileNameFormat;
return this;
}
- public AvroGenericRecordBolt withSyncPolicy(SyncPolicy syncPolicy){
+ public AvroGenericRecordBolt withSyncPolicy(SyncPolicy syncPolicy) {
this.syncPolicy = syncPolicy;
return this;
}
- public AvroGenericRecordBolt withRotationPolicy(FileRotationPolicy rotationPolicy){
+ public AvroGenericRecordBolt withRotationPolicy(FileRotationPolicy rotationPolicy) {
this.rotationPolicy = rotationPolicy;
return this;
}
- public AvroGenericRecordBolt addRotationAction(RotationAction action){
+ public AvroGenericRecordBolt addRotationAction(RotationAction action) {
this.rotationActions.add(action);
return this;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
index ba8b24e..6677c7b 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
@@ -1,25 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Tuple;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Map;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -31,51 +26,49 @@ import org.apache.storm.hdfs.common.AbstractHDFSWriter;
import org.apache.storm.hdfs.common.HDFSWriter;
import org.apache.storm.hdfs.common.Partitioner;
import org.apache.storm.hdfs.common.rotation.RotationAction;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.net.URI;
-import java.util.EnumSet;
-import java.util.Map;
-
-public class HdfsBolt extends AbstractHdfsBolt{
+public class HdfsBolt extends AbstractHdfsBolt {
private static final Logger LOG = LoggerFactory.getLogger(HdfsBolt.class);
private transient FSDataOutputStream out;
private RecordFormat format;
- public HdfsBolt withFsUrl(String fsUrl){
+ public HdfsBolt withFsUrl(String fsUrl) {
this.fsUrl = fsUrl;
return this;
}
- public HdfsBolt withConfigKey(String configKey){
+ public HdfsBolt withConfigKey(String configKey) {
this.configKey = configKey;
return this;
}
- public HdfsBolt withFileNameFormat(FileNameFormat fileNameFormat){
+ public HdfsBolt withFileNameFormat(FileNameFormat fileNameFormat) {
this.fileNameFormat = fileNameFormat;
return this;
}
- public HdfsBolt withRecordFormat(RecordFormat format){
+ public HdfsBolt withRecordFormat(RecordFormat format) {
this.format = format;
return this;
}
- public HdfsBolt withSyncPolicy(SyncPolicy syncPolicy){
+ public HdfsBolt withSyncPolicy(SyncPolicy syncPolicy) {
this.syncPolicy = syncPolicy;
return this;
}
- public HdfsBolt withRotationPolicy(FileRotationPolicy rotationPolicy){
+ public HdfsBolt withRotationPolicy(FileRotationPolicy rotationPolicy) {
this.rotationPolicy = rotationPolicy;
return this;
}
- public HdfsBolt addRotationAction(RotationAction action){
+ public HdfsBolt addRotationAction(RotationAction action) {
this.rotationActions.add(action);
return this;
}
@@ -114,6 +107,6 @@ public class HdfsBolt extends AbstractHdfsBolt{
@Override
protected AbstractHDFSWriter makeNewWriter(Path path, Tuple tuple) throws IOException {
this.out = this.fs.create(path);
- return new HDFSWriter(rotationPolicy,path, out, format);
+ return new HDFSWriter(rotationPolicy, path, out, format);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
index b2120dc..c73c6a2 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
@@ -1,25 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Tuple;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Map;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
@@ -32,13 +27,12 @@ import org.apache.storm.hdfs.common.AbstractHDFSWriter;
import org.apache.storm.hdfs.common.Partitioner;
import org.apache.storm.hdfs.common.SequenceFileWriter;
import org.apache.storm.hdfs.common.rotation.RotationAction;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.net.URI;
-import java.util.Map;
-
public class SequenceFileBolt extends AbstractHdfsBolt {
private static final Logger LOG = LoggerFactory.getLogger(SequenceFileBolt.class);
@@ -52,7 +46,7 @@ public class SequenceFileBolt extends AbstractHdfsBolt {
public SequenceFileBolt() {
}
- public SequenceFileBolt withCompressionCodec(String codec){
+ public SequenceFileBolt withCompressionCodec(String codec) {
this.compressionCodec = codec;
return this;
}
@@ -62,7 +56,7 @@ public class SequenceFileBolt extends AbstractHdfsBolt {
return this;
}
- public SequenceFileBolt withConfigKey(String configKey){
+ public SequenceFileBolt withConfigKey(String configKey) {
this.configKey = configKey;
return this;
}
@@ -87,7 +81,7 @@ public class SequenceFileBolt extends AbstractHdfsBolt {
return this;
}
- public SequenceFileBolt withCompressionType(SequenceFile.CompressionType compressionType){
+ public SequenceFileBolt withCompressionType(SequenceFile.CompressionType compressionType) {
this.compressionType = compressionType;
return this;
}
@@ -97,7 +91,7 @@ public class SequenceFileBolt extends AbstractHdfsBolt {
return this;
}
- public SequenceFileBolt addRotationAction(RotationAction action){
+ public SequenceFileBolt addRotationAction(RotationAction action) {
this.rotationActions.add(action);
return this;
}
@@ -134,11 +128,11 @@ public class SequenceFileBolt extends AbstractHdfsBolt {
@Override
protected AbstractHDFSWriter makeNewWriter(Path path, Tuple tuple) throws IOException {
SequenceFile.Writer writer = SequenceFile.createWriter(
- this.hdfsConfig,
- SequenceFile.Writer.file(path),
- SequenceFile.Writer.keyClass(this.format.keyClass()),
- SequenceFile.Writer.valueClass(this.format.valueClass()),
- SequenceFile.Writer.compression(this.compressionType, this.codecFactory.getCodecByName(this.compressionCodec))
+ this.hdfsConfig,
+ SequenceFile.Writer.file(path),
+ SequenceFile.Writer.keyClass(this.format.keyClass()),
+ SequenceFile.Writer.valueClass(this.format.valueClass()),
+ SequenceFile.Writer.compression(this.compressionType, this.codecFactory.getCodecByName(this.compressionCodec))
);
return new SequenceFileWriter(this.rotationPolicy, path, writer, this.format);
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java
index 9312f2c..c6b3fb3 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/Writer.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.bolt;
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java
index a3afa18..14b4513 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultFileNameFormat.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.bolt.format;
-import org.apache.storm.task.TopologyContext;
+package org.apache.storm.hdfs.bolt.format;
import java.util.Map;
+import org.apache.storm.task.TopologyContext;
/**
@@ -48,7 +42,7 @@ public class DefaultFileNameFormat implements FileNameFormat {
* @param prefix
* @return
*/
- public DefaultFileNameFormat withPrefix(String prefix){
+ public DefaultFileNameFormat withPrefix(String prefix) {
this.prefix = prefix;
return this;
}
@@ -59,12 +53,12 @@ public class DefaultFileNameFormat implements FileNameFormat {
* @param extension
* @return
*/
- public DefaultFileNameFormat withExtension(String extension){
+ public DefaultFileNameFormat withExtension(String extension) {
this.extension = extension;
return this;
}
- public DefaultFileNameFormat withPath(String path){
+ public DefaultFileNameFormat withPath(String path) {
this.path = path;
return this;
}
@@ -77,10 +71,10 @@ public class DefaultFileNameFormat implements FileNameFormat {
@Override
public String getName(long rotation, long timeStamp) {
- return this.prefix + this.componentId + "-" + this.taskId + "-" + rotation + "-" + timeStamp + this.extension;
+ return this.prefix + this.componentId + "-" + this.taskId + "-" + rotation + "-" + timeStamp + this.extension;
}
- public String getPath(){
+ public String getPath() {
return this.path;
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultSequenceFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultSequenceFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultSequenceFormat.java
index ab07d43..8db7ffc 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultSequenceFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DefaultSequenceFormat.java
@@ -15,17 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt.format;
-import org.apache.storm.tuple.Tuple;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
+import org.apache.storm.tuple.Tuple;
/**
* Basic <code>SequenceFormat</code> implementation that uses
* <code>LongWritable</code> for keys and <code>Text</code> for values.
- *
*/
public class DefaultSequenceFormat implements SequenceFormat {
private transient LongWritable key;
@@ -34,7 +34,7 @@ public class DefaultSequenceFormat implements SequenceFormat {
private String keyField;
private String valueField;
- public DefaultSequenceFormat(String keyField, String valueField){
+ public DefaultSequenceFormat(String keyField, String valueField) {
this.keyField = keyField;
this.valueField = valueField;
}
@@ -51,8 +51,8 @@ public class DefaultSequenceFormat implements SequenceFormat {
@Override
public Writable key(Tuple tuple) {
- if(this.key == null){
- this.key = new LongWritable();
+ if (this.key == null) {
+ this.key = new LongWritable();
}
this.key.set(tuple.getLongByField(this.keyField));
return this.key;
@@ -60,7 +60,7 @@ public class DefaultSequenceFormat implements SequenceFormat {
@Override
public Writable value(Tuple tuple) {
- if(this.value == null){
+ if (this.value == null) {
this.value = new Text();
}
this.value.set(tuple.getStringByField(this.valueField));
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java
index 0c6e3f0..e9a81cc 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/DelimitedRecordFormat.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt.format;
import org.apache.storm.tuple.Fields;
@@ -44,7 +39,7 @@ public class DelimitedRecordFormat implements RecordFormat {
* @param fields
* @return
*/
- public DelimitedRecordFormat withFields(Fields fields){
+ public DelimitedRecordFormat withFields(Fields fields) {
this.fields = fields;
return this;
}
@@ -55,7 +50,7 @@ public class DelimitedRecordFormat implements RecordFormat {
* @param delimiter
* @return
*/
- public DelimitedRecordFormat withFieldDelimiter(String delimiter){
+ public DelimitedRecordFormat withFieldDelimiter(String delimiter) {
this.fieldDelimiter = delimiter;
return this;
}
@@ -66,7 +61,7 @@ public class DelimitedRecordFormat implements RecordFormat {
* @param delimiter
* @return
*/
- public DelimitedRecordFormat withRecordDelimiter(String delimiter){
+ public DelimitedRecordFormat withRecordDelimiter(String delimiter) {
this.recordDelimiter = delimiter;
return this;
}
@@ -76,9 +71,9 @@ public class DelimitedRecordFormat implements RecordFormat {
StringBuilder sb = new StringBuilder();
Fields fields = this.fields == null ? tuple.getFields() : this.fields;
int size = fields.size();
- for(int i = 0; i < size; i++){
+ for (int i = 0; i < size; i++) {
sb.append(tuple.getValueByField(fields.get(i)));
- if(i != size - 1){
+ if (i != size - 1) {
sb.append(this.fieldDelimiter);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java
index 9ef2851..891c600 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/FileNameFormat.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.bolt.format;
-import org.apache.storm.task.TopologyContext;
+package org.apache.storm.hdfs.bolt.format;
import java.io.Serializable;
import java.util.Map;
+import org.apache.storm.task.TopologyContext;
/**
* Formatter interface for determining HDFS file names.
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java
index fe48f05..8d55c03 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/RecordFormat.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.bolt.format;
+package org.apache.storm.hdfs.bolt.format;
-import org.apache.storm.tuple.Tuple;
import java.io.Serializable;
+import org.apache.storm.tuple.Tuple;
/**
* Formats a Tuple object into a byte array
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SequenceFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SequenceFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SequenceFormat.java
index fcb7f45..7c38f66 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SequenceFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SequenceFormat.java
@@ -15,15 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.hdfs.bolt.format;
-import org.apache.storm.tuple.Tuple;
+package org.apache.storm.hdfs.bolt.format;
import java.io.Serializable;
+import org.apache.storm.tuple.Tuple;
/**
* Interface for converting <code>Tuple</code> objects to HDFS sequence file key-value pairs.
- *
*/
public interface SequenceFormat extends Serializable {
/**
@@ -35,6 +34,7 @@ public interface SequenceFormat extends Serializable {
/**
* Value class used by implementation (e.g. Text.class, etc.)
+ *
* @return
*/
Class valueClass();
@@ -49,6 +49,7 @@ public interface SequenceFormat extends Serializable {
/**
* Given a tuple, return the value that should be written to the sequence file.
+ *
* @param tuple
* @return
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java
index cb37bdc..7869d69 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/format/SimpleFileNameFormat.java
@@ -1,27 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt.format;
import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Map;
-
import org.apache.storm.task.TopologyContext;
import org.apache.storm.utils.Utils;
@@ -41,11 +35,11 @@ public class SimpleFileNameFormat implements FileNameFormat {
// compile parameters
SimpleDateFormat dateFormat = new SimpleDateFormat(timeFormat);
String ret = name
- .replace("$TIME", dateFormat.format(new Date(timeStamp)))
- .replace("$NUM", String.valueOf(rotation))
- .replace("$HOST", host)
- .replace("$COMPONENT", componentId)
- .replace("$TASK", String.valueOf(taskId));
+ .replace("$TIME", dateFormat.format(new Date(timeStamp)))
+ .replace("$NUM", String.valueOf(rotation))
+ .replace("$HOST", host)
+ .replace("$COMPONENT", componentId)
+ .replace("$TASK", String.valueOf(taskId));
return ret;
}
@@ -78,7 +72,7 @@ public class SimpleFileNameFormat implements FileNameFormat {
* $HOST - local host name<br/>
* $COMPONENT - component id<br/>
* $TASK - task id<br/>
- *
+ *
* @param name
* file name
* @return
@@ -90,10 +84,10 @@ public class SimpleFileNameFormat implements FileNameFormat {
public SimpleFileNameFormat withTimeFormat(String timeFormat) {
//check format
- try{
+ try {
new SimpleDateFormat(timeFormat);
- }catch (Exception e) {
- throw new IllegalArgumentException("invalid timeFormat: "+e.getMessage());
+ } catch (Exception e) {
+ throw new IllegalArgumentException("invalid timeFormat: " + e.getMessage());
}
this.timeFormat = timeFormat;
return this;
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java
index aeb63fa..6354ae7 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileRotationPolicy.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.bolt.rotation;
+package org.apache.storm.hdfs.bolt.rotation;
-import org.apache.storm.tuple.Tuple;
import java.io.Serializable;
+import org.apache.storm.tuple.Tuple;
/**
* Used by the HdfsBolt to decide when to rotate files.
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java
index 5fb9bbc..230e8b4 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/FileSizeRotationPolicy.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt.rotation;
@@ -36,32 +31,11 @@ import org.slf4j.LoggerFactory;
*/
public class FileSizeRotationPolicy implements FileRotationPolicy {
private static final Logger LOG = LoggerFactory.getLogger(FileSizeRotationPolicy.class);
-
- public static enum Units {
-
- KB((long)Math.pow(2, 10)),
- MB((long)Math.pow(2, 20)),
- GB((long)Math.pow(2, 30)),
- TB((long)Math.pow(2, 40));
-
- private long byteCount;
-
- private Units(long byteCount){
- this.byteCount = byteCount;
- }
-
- public long getByteCount(){
- return byteCount;
- }
- }
-
private long maxBytes;
-
private long lastOffset = 0;
private long currentBytesWritten = 0;
-
- public FileSizeRotationPolicy(float count, Units units){
- this.maxBytes = (long)(count * units.getByteCount());
+ public FileSizeRotationPolicy(float count, Units units) {
+ this.maxBytes = (long) (count * units.getByteCount());
}
protected FileSizeRotationPolicy(long maxBytes) {
@@ -86,4 +60,22 @@ public class FileSizeRotationPolicy implements FileRotationPolicy {
public FileRotationPolicy copy() {
return new FileSizeRotationPolicy(this.maxBytes);
}
+
+ public static enum Units {
+
+ KB((long) Math.pow(2, 10)),
+ MB((long) Math.pow(2, 20)),
+ GB((long) Math.pow(2, 30)),
+ TB((long) Math.pow(2, 40));
+
+ private long byteCount;
+
+ private Units(long byteCount) {
+ this.byteCount = byteCount;
+ }
+
+ public long getByteCount() {
+ return byteCount;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java
index a00037b..f25be14 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/NoRotationPolicy.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt.rotation;
import org.apache.storm.tuple.Tuple;
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java
index 06fada8..e600831 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java
@@ -15,34 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt.rotation;
import org.apache.storm.tuple.Tuple;
public class TimedRotationPolicy implements FileRotationPolicy {
- public static enum TimeUnit {
-
- SECONDS((long)1000),
- MINUTES((long)1000*60),
- HOURS((long)1000*60*60),
- DAYS((long)1000*60*60*24);
-
- private long milliSeconds;
-
- private TimeUnit(long milliSeconds){
- this.milliSeconds = milliSeconds;
- }
-
- public long getMilliSeconds(){
- return milliSeconds;
- }
- }
-
private long interval;
- public TimedRotationPolicy(float count, TimeUnit units){
- this.interval = (long)(count * units.getMilliSeconds());
+ public TimedRotationPolicy(float count, TimeUnit units) {
+ this.interval = (long) (count * units.getMilliSeconds());
}
protected TimedRotationPolicy(long interval) {
@@ -74,7 +57,25 @@ public class TimedRotationPolicy implements FileRotationPolicy {
return new TimedRotationPolicy(this.interval);
}
- public long getInterval(){
+ public long getInterval() {
return this.interval;
}
+
+ public static enum TimeUnit {
+
+ SECONDS((long) 1000),
+ MINUTES((long) 1000 * 60),
+ HOURS((long) 1000 * 60 * 60),
+ DAYS((long) 1000 * 60 * 60 * 24);
+
+ private long milliSeconds;
+
+ private TimeUnit(long milliSeconds) {
+ this.milliSeconds = milliSeconds;
+ }
+
+ public long getMilliSeconds() {
+ return milliSeconds;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java
index 9f31d58..45abc7d 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/CountSyncPolicy.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.bolt.sync;
@@ -29,7 +24,7 @@ public class CountSyncPolicy implements SyncPolicy {
private int count;
private int executeCount = 0;
- public CountSyncPolicy(int count){
+ public CountSyncPolicy(int count) {
this.count = count;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java
index 19c0e25..12c673f 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/sync/SyncPolicy.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.bolt.sync;
-import org.apache.storm.tuple.Tuple;
+package org.apache.storm.hdfs.bolt.sync;
import java.io.Serializable;
+import org.apache.storm.tuple.Tuple;
/**
* Interface for controlling when the HdfsBolt
[31/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-cassandra
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/trident/MapStateTest.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/trident/MapStateTest.java b/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/trident/MapStateTest.java
index 3c10dd9..b47165b 100644
--- a/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/trident/MapStateTest.java
+++ b/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/trident/MapStateTest.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.trident;
import com.datastax.driver.core.Cluster;
@@ -25,6 +19,8 @@ import com.datastax.driver.core.querybuilder.QueryBuilder;
import com.datastax.driver.core.querybuilder.Truncate;
import com.datastax.driver.core.schemabuilder.Create;
import com.datastax.driver.core.schemabuilder.SchemaBuilder;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
@@ -43,14 +39,13 @@ import org.apache.storm.trident.testing.FixedBatchSpout;
import org.apache.storm.trident.testing.Split;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.junit.*;
-
-import java.util.HashMap;
-import java.util.Map;
-
import static org.junit.Assert.assertEquals;
public class MapStateTest {
@@ -62,13 +57,20 @@ public class MapStateTest {
private static Cluster cluster;
private Session session;
+ protected static Column column(String name, DataType type) {
+ Column column = new Column();
+ column.name = name;
+ column.type = type;
+ return column;
+ }
+
@Test
public void nonTransactionalStateTest() throws Exception {
StateFactory factory = MapStateFactoryBuilder.nontransactional(getCassandraConfig())
- .withTable("words_ks", "words_table")
- .withKeys("word")
- .withJSONBinaryState("state")
- .build();
+ .withTable("words_ks", "words_table")
+ .withKeys("word")
+ .withJSONBinaryState("state")
+ .build();
wordsTest(factory);
}
@@ -78,10 +80,10 @@ public class MapStateTest {
Map<String, Object> config = new HashMap();
StateFactory factory = MapStateFactoryBuilder.transactional(getCassandraConfig())
- .withTable("words_ks", "words_table")
- .withKeys("word")
- .withJSONBinaryState("state")
- .build();
+ .withTable("words_ks", "words_table")
+ .withKeys("word")
+ .withJSONBinaryState("state")
+ .build();
wordsTest(factory);
}
@@ -91,10 +93,10 @@ public class MapStateTest {
Map<String, Object> config = new HashMap();
StateFactory factory = MapStateFactoryBuilder.opaque(getCassandraConfig())
- .withTable("words_ks", "words_table")
- .withKeys("word")
- .withJSONBinaryState("state")
- .build();
+ .withTable("words_ks", "words_table")
+ .withKeys("word")
+ .withJSONBinaryState("state")
+ .build();
wordsTest(factory);
}
@@ -102,20 +104,20 @@ public class MapStateTest {
public void wordsTest(StateFactory factory) throws Exception {
FixedBatchSpout spout = new FixedBatchSpout(
- new Fields("sentence"), 3,
- new Values("the cow jumped over the moon"),
- new Values("the man went to the store and bought some candy"),
- new Values("four score and seven years ago"),
- new Values("how many apples can you eat"));
+ new Fields("sentence"), 3,
+ new Values("the cow jumped over the moon"),
+ new Values("the man went to the store and bought some candy"),
+ new Values("four score and seven years ago"),
+ new Values("how many apples can you eat"));
spout.setCycle(false);
TridentTopology topology = new TridentTopology();
TridentState wordCounts = topology.newStream("spout1", spout)
- .each(new Fields("sentence"), new Split(), new Fields("word"))
- .groupBy(new Fields("word"))
- .persistentAggregate(factory, new Count(), new Fields("state"))
- .parallelismHint(1);
+ .each(new Fields("sentence"), new Split(), new Fields("word"))
+ .groupBy(new Fields("word"))
+ .persistentAggregate(factory, new Count(), new Fields("state"))
+ .parallelismHint(1);
LocalDRPC client = new LocalDRPC();
topology.newDRPCStream("words", client)
@@ -134,7 +136,7 @@ public class MapStateTest {
do {
Thread.sleep(2000);
count = session.execute(QueryBuilder.select().all().from("words_ks", "words_table"))
- .getAvailableWithoutFetching();
+ .getAvailableWithoutFetching();
logger.info("Found {} records", count);
} while (count < 24);
@@ -164,8 +166,8 @@ public class MapStateTest {
createKeyspace("words_ks");
createTable("words_ks", "words_table",
- column("word", DataType.varchar()),
- column("state", DataType.blob()));
+ column("word", DataType.varchar()),
+ column("state", DataType.blob()));
}
@@ -178,11 +180,11 @@ public class MapStateTest {
protected void createKeyspace(String keyspace) throws Exception {
// Create keyspace not supported in the current datastax driver
String createKeyspace = "CREATE KEYSPACE IF NOT EXISTS "
- + keyspace
- + " WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };";
+ + keyspace
+ + " WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };";
logger.info(createKeyspace);
if (!session.execute(createKeyspace)
- .wasApplied()) {
+ .wasApplied()) {
throw new Exception("Did not create keyspace " + keyspace);
}
}
@@ -206,8 +208,8 @@ public class MapStateTest {
replication.put("replication_factor", 1);
Create createTable = SchemaBuilder.createTable(keyspace, table)
- .ifNotExists()
- .addPartitionKey(key.name, key.type);
+ .ifNotExists()
+ .addPartitionKey(key.name, key.type);
for (Column field : fields) {
createTable.addColumn(field.name, field.type);
}
@@ -215,13 +217,6 @@ public class MapStateTest {
session.execute(createTable);
}
- protected static Column column(String name, DataType type) {
- Column column = new Column();
- column.name = name;
- column.type = type;
- return column;
- }
-
protected static class Column {
public String name;
public DataType type;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/trident/WeatherBatchSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/trident/WeatherBatchSpout.java b/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/trident/WeatherBatchSpout.java
index 6aaed56..736b6a8 100644
--- a/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/trident/WeatherBatchSpout.java
+++ b/external/storm-cassandra/src/test/java/org/apache/storm/cassandra/trident/WeatherBatchSpout.java
@@ -1,37 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.cassandra.trident;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Fields;
-import com.google.common.collect.Lists;
-import org.apache.storm.trident.operation.TridentCollector;
-import org.apache.storm.trident.spout.IBatchSpout;
+package org.apache.storm.cassandra.trident;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.UUID;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.spout.IBatchSpout;
+import org.apache.storm.tuple.Fields;
/**
*
@@ -57,24 +47,25 @@ public class WeatherBatchSpout implements IBatchSpout {
@Override
public void emitBatch(long batchId, TridentCollector collector) {
List<List<Object>> batch = this.batches.get(batchId);
- if(batch == null){
+ if (batch == null) {
batch = new ArrayList<>();
- for (int i=0; i< batchSize; i++) {
+ for (int i = 0; i < batchSize; i++) {
batch.add(createTuple());
}
this.batches.put(batchId, batch);
}
- for(List<Object> list : batch){
+ for (List<Object> list : batch) {
collector.emit(list);
}
}
private List<Object> createTuple() {
final Random random = new Random();
- List<Object> values = new ArrayList<Object>(){{
+ List<Object> values = new ArrayList<Object>() {{
add(stationIds[random.nextInt(stationIds.length)]);
add(random.nextInt(100) + "");
- add(UUID.randomUUID());}};
+ add(UUID.randomUUID());
+ }};
return values;
}
[08/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/Node.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/Node.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/Node.java
index 389faba..89fe462 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/Node.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/Node.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.multitenant;
@@ -23,318 +17,313 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
-import java.util.Set;
import java.util.Map.Entry;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import java.util.Set;
import org.apache.storm.scheduler.Cluster;
import org.apache.storm.scheduler.ExecutorDetails;
import org.apache.storm.scheduler.SchedulerAssignment;
import org.apache.storm.scheduler.SupervisorDetails;
import org.apache.storm.scheduler.WorkerSlot;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Represents a single node in the cluster.
*/
public class Node {
- private static final Logger LOG = LoggerFactory.getLogger(Node.class);
- private Map<String, Set<WorkerSlot>> _topIdToUsedSlots = new HashMap<>();
- private Set<WorkerSlot> _freeSlots = new HashSet<>();
- private final String _nodeId;
- private boolean _isAlive;
-
- public Node(String nodeId, Set<Integer> allPorts, boolean isAlive) {
- _nodeId = nodeId;
- _isAlive = isAlive;
- if (_isAlive && allPorts != null) {
- for (int port: allPorts) {
- _freeSlots.add(new WorkerSlot(_nodeId, port));
- }
- }
- }
+ /**
+ * Used to sort a list of nodes so the node with the most free slots comes
+ * first.
+ */
+ public static final Comparator<Node> FREE_NODE_COMPARATOR_DEC = new Comparator<Node>() {
+ @Override
+ public int compare(Node o1, Node o2) {
+ return o1.totalSlotsUsed() - o2.totalSlotsUsed();
+ }
+ };
+ private static final Logger LOG = LoggerFactory.getLogger(Node.class);
+ private final String _nodeId;
+ private Map<String, Set<WorkerSlot>> _topIdToUsedSlots = new HashMap<>();
+ private Set<WorkerSlot> _freeSlots = new HashSet<>();
+ private boolean _isAlive;
- public String getId() {
- return _nodeId;
- }
-
- public boolean isAlive() {
- return _isAlive;
- }
-
- /**
- * @return a collection of the topology ids currently running on this node
- */
- public Collection<String> getRunningTopologies() {
- return _topIdToUsedSlots.keySet();
- }
-
- public boolean isTotallyFree() {
- return _topIdToUsedSlots.isEmpty();
- }
-
- public int totalSlotsFree() {
- return _freeSlots.size();
- }
-
- public int totalSlotsUsed() {
- int total = 0;
- for (Set<WorkerSlot> slots: _topIdToUsedSlots.values()) {
- total += slots.size();
- }
- return total;
- }
-
- public int totalSlots() {
- return totalSlotsFree() + totalSlotsUsed();
- }
-
- public int totalSlotsUsed(String topId) {
- int total = 0;
- Set<WorkerSlot> slots = _topIdToUsedSlots.get(topId);
- if (slots != null) {
- total = slots.size();
+ public Node(String nodeId, Set<Integer> allPorts, boolean isAlive) {
+ _nodeId = nodeId;
+ _isAlive = isAlive;
+ if (_isAlive && allPorts != null) {
+ for (int port : allPorts) {
+ _freeSlots.add(new WorkerSlot(_nodeId, port));
+ }
+ }
}
- return total;
- }
- private void validateSlot(WorkerSlot ws) {
- if (!_nodeId.equals(ws.getNodeId())) {
- throw new IllegalArgumentException(
- "Trying to add a slot to the wrong node " + ws +
- " is not a part of " + _nodeId);
+ public static int countSlotsUsed(String topId, Collection<Node> nodes) {
+ int total = 0;
+ for (Node n : nodes) {
+ total += n.totalSlotsUsed(topId);
+ }
+ return total;
}
- }
-
- private void addOrphanedSlot(WorkerSlot ws) {
- if (_isAlive) {
- throw new IllegalArgumentException("Orphaned Slots " +
- "only are allowed on dead nodes.");
+
+ public static int countSlotsUsed(Collection<Node> nodes) {
+ int total = 0;
+ for (Node n : nodes) {
+ total += n.totalSlotsUsed();
+ }
+ return total;
}
- validateSlot(ws);
- if (_freeSlots.contains(ws)) {
- return;
+
+ public static int countFreeSlotsAlive(Collection<Node> nodes) {
+ int total = 0;
+ for (Node n : nodes) {
+ if (n.isAlive()) {
+ total += n.totalSlotsFree();
+ }
+ }
+ return total;
}
- for (Set<WorkerSlot> used: _topIdToUsedSlots.values()) {
- if (used.contains(ws)) {
- return;
- }
+
+ public static int countTotalSlotsAlive(Collection<Node> nodes) {
+ int total = 0;
+ for (Node n : nodes) {
+ if (n.isAlive()) {
+ total += n.totalSlots();
+ }
+ }
+ return total;
}
- _freeSlots.add(ws);
- }
-
- boolean assignInternal(WorkerSlot ws, String topId, boolean dontThrow) {
- validateSlot(ws);
- if (!_freeSlots.remove(ws)) {
- for (Entry<String, Set<WorkerSlot>> topologySetEntry : _topIdToUsedSlots.entrySet()) {
- if (topologySetEntry.getValue().contains(ws)) {
- if (dontThrow) {
- LOG.warn("Worker slot [" + ws + "] can't be assigned to " + topId +
- ". Its already assigned to " + topologySetEntry.getKey() + ".");
- return true;
- }
- throw new IllegalStateException("Worker slot [" + ws + "] can't be assigned to "
- + topId + ". Its already assigned to " + topologySetEntry.getKey() + ".");
+
+ public static Map<String, Node> getAllNodesFrom(Cluster cluster) {
+ Map<String, Node> nodeIdToNode = new HashMap<>();
+ for (SupervisorDetails sup : cluster.getSupervisors().values()) {
+ //Node ID and supervisor ID are the same.
+ String id = sup.getId();
+ boolean isAlive = !cluster.isBlackListed(id);
+ LOG.debug("Found a {} Node {} {}",
+ isAlive ? "living" : "dead", id, sup.getAllPorts());
+ nodeIdToNode.put(id, new Node(id, sup.getAllPorts(), isAlive));
}
- }
- LOG.warn("Adding Worker slot [" + ws + "] that was not reported in the supervisor heartbeats," +
- " but the worker is already running for topology " + topId + ".");
+
+ for (Entry<String, SchedulerAssignment> entry : cluster.getAssignments().entrySet()) {
+ String topId = entry.getValue().getTopologyId();
+ for (WorkerSlot ws : entry.getValue().getSlots()) {
+ String id = ws.getNodeId();
+ Node node = nodeIdToNode.get(id);
+ if (node == null) {
+ LOG.debug("Found an assigned slot on a dead supervisor {}", ws);
+ node = new Node(id, null, false);
+ nodeIdToNode.put(id, node);
+ }
+ if (!node.isAlive()) {
+ //The supervisor on the node down so add an orphaned slot to hold the unsupervised worker
+ node.addOrphanedSlot(ws);
+ }
+ if (node.assignInternal(ws, topId, true)) {
+ LOG.warn("Bad scheduling state for topology [" + topId + "], the slot " +
+ ws + " assigned to multiple workers, un-assigning everything...");
+ node.free(ws, cluster, true);
+ }
+ }
+ }
+
+ return nodeIdToNode;
}
- Set<WorkerSlot> usedSlots = _topIdToUsedSlots.get(topId);
- if (usedSlots == null) {
- usedSlots = new HashSet<>();
- _topIdToUsedSlots.put(topId, usedSlots);
+
+ public String getId() {
+ return _nodeId;
}
- usedSlots.add(ws);
- return false;
- }
-
- /**
- * Free all slots on this node. This will update the Cluster too.
- * @param cluster the cluster to be updated
- */
- public void freeAllSlots(Cluster cluster) {
- if (!_isAlive) {
- LOG.warn("Freeing all slots on a dead node {} ",_nodeId);
- }
- for (Entry<String, Set<WorkerSlot>> entry : _topIdToUsedSlots.entrySet()) {
- cluster.freeSlots(entry.getValue());
- if (_isAlive) {
- _freeSlots.addAll(entry.getValue());
- }
+
+ public boolean isAlive() {
+ return _isAlive;
}
- _topIdToUsedSlots = new HashMap<>();
- }
-
- /**
- * Frees a single slot in this node
- * @param ws the slot to free
- * @param cluster the cluster to update
- */
- public void free(WorkerSlot ws, Cluster cluster, boolean forceFree) {
- if (_freeSlots.contains(ws)) return;
- boolean wasFound = false;
- for (Entry<String, Set<WorkerSlot>> entry : _topIdToUsedSlots.entrySet()) {
- Set<WorkerSlot> slots = entry.getValue();
- if (slots.remove(ws)) {
- cluster.freeSlot(ws);
- if (_isAlive) {
- _freeSlots.add(ws);
- }
- wasFound = true;
- }
+
+ /**
+ * @return a collection of the topology ids currently running on this node
+ */
+ public Collection<String> getRunningTopologies() {
+ return _topIdToUsedSlots.keySet();
}
- if(!wasFound)
- {
- if(forceFree)
- {
- LOG.info("Forcefully freeing the " + ws);
- cluster.freeSlot(ws);
- _freeSlots.add(ws);
- } else {
- throw new IllegalArgumentException("Tried to free a slot that was not" +
- " part of this node " + _nodeId);
- }
+
+ public boolean isTotallyFree() {
+ return _topIdToUsedSlots.isEmpty();
}
- }
-
- /**
- * Frees all the slots for a topology.
- * @param topId the topology to free slots for
- * @param cluster the cluster to update
- */
- public void freeTopology(String topId, Cluster cluster) {
- Set<WorkerSlot> slots = _topIdToUsedSlots.get(topId);
- if (slots == null || slots.isEmpty()) return;
- for (WorkerSlot ws : slots) {
- cluster.freeSlot(ws);
- if (_isAlive) {
- _freeSlots.add(ws);
- }
+
+ public int totalSlotsFree() {
+ return _freeSlots.size();
}
- _topIdToUsedSlots.remove(topId);
- }
-
- /**
- * Assign a free slot on the node to the following topology and executors.
- * This will update the cluster too.
- * @param topId the topology to assign a free slot to.
- * @param executors the executors to run in that slot.
- * @param cluster the cluster to be updated
- */
- public void assign(String topId, Collection<ExecutorDetails> executors,
- Cluster cluster) {
- if (!_isAlive) {
- throw new IllegalStateException("Trying to adding to a dead node " + _nodeId);
+
+ public int totalSlotsUsed() {
+ int total = 0;
+ for (Set<WorkerSlot> slots : _topIdToUsedSlots.values()) {
+ total += slots.size();
+ }
+ return total;
+ }
+
+ public int totalSlots() {
+ return totalSlotsFree() + totalSlotsUsed();
}
- if (_freeSlots.isEmpty()) {
- throw new IllegalStateException("Trying to assign to a full node " + _nodeId);
+
+ public int totalSlotsUsed(String topId) {
+ int total = 0;
+ Set<WorkerSlot> slots = _topIdToUsedSlots.get(topId);
+ if (slots != null) {
+ total = slots.size();
+ }
+ return total;
}
- if (executors.size() == 0) {
- LOG.warn("Trying to assign nothing from " + topId + " to " + _nodeId + " (Ignored)");
- } else {
- WorkerSlot slot = _freeSlots.iterator().next();
- cluster.assign(slot, topId, executors);
- assignInternal(slot, topId, false);
+
+ private void validateSlot(WorkerSlot ws) {
+ if (!_nodeId.equals(ws.getNodeId())) {
+ throw new IllegalArgumentException(
+ "Trying to add a slot to the wrong node " + ws +
+ " is not a part of " + _nodeId);
+ }
}
- }
-
- @Override
- public boolean equals(Object other) {
- return other instanceof Node && _nodeId.equals(((Node) other)._nodeId);
- }
-
- @Override
- public int hashCode() {
- return _nodeId.hashCode();
- }
-
- @Override
- public String toString() {
- return "Node: " + _nodeId;
- }
- public static int countSlotsUsed(String topId, Collection<Node> nodes) {
- int total = 0;
- for (Node n: nodes) {
- total += n.totalSlotsUsed(topId);
+ private void addOrphanedSlot(WorkerSlot ws) {
+ if (_isAlive) {
+ throw new IllegalArgumentException("Orphaned Slots " +
+ "only are allowed on dead nodes.");
+ }
+ validateSlot(ws);
+ if (_freeSlots.contains(ws)) {
+ return;
+ }
+ for (Set<WorkerSlot> used : _topIdToUsedSlots.values()) {
+ if (used.contains(ws)) {
+ return;
+ }
+ }
+ _freeSlots.add(ws);
}
- return total;
- }
-
- public static int countSlotsUsed(Collection<Node> nodes) {
- int total = 0;
- for (Node n: nodes) {
- total += n.totalSlotsUsed();
+
+ boolean assignInternal(WorkerSlot ws, String topId, boolean dontThrow) {
+ validateSlot(ws);
+ if (!_freeSlots.remove(ws)) {
+ for (Entry<String, Set<WorkerSlot>> topologySetEntry : _topIdToUsedSlots.entrySet()) {
+ if (topologySetEntry.getValue().contains(ws)) {
+ if (dontThrow) {
+ LOG.warn("Worker slot [" + ws + "] can't be assigned to " + topId +
+ ". Its already assigned to " + topologySetEntry.getKey() + ".");
+ return true;
+ }
+ throw new IllegalStateException("Worker slot [" + ws + "] can't be assigned to "
+ + topId + ". Its already assigned to " + topologySetEntry.getKey() + ".");
+ }
+ }
+ LOG.warn("Adding Worker slot [" + ws + "] that was not reported in the supervisor heartbeats," +
+ " but the worker is already running for topology " + topId + ".");
+ }
+ Set<WorkerSlot> usedSlots = _topIdToUsedSlots.get(topId);
+ if (usedSlots == null) {
+ usedSlots = new HashSet<>();
+ _topIdToUsedSlots.put(topId, usedSlots);
+ }
+ usedSlots.add(ws);
+ return false;
}
- return total;
- }
-
- public static int countFreeSlotsAlive(Collection<Node> nodes) {
- int total = 0;
- for (Node n: nodes) {
- if (n.isAlive()) {
- total += n.totalSlotsFree();
- }
+
+ /**
+ * Free all slots on this node. This will update the Cluster too.
+ * @param cluster the cluster to be updated
+ */
+ public void freeAllSlots(Cluster cluster) {
+ if (!_isAlive) {
+ LOG.warn("Freeing all slots on a dead node {} ", _nodeId);
+ }
+ for (Entry<String, Set<WorkerSlot>> entry : _topIdToUsedSlots.entrySet()) {
+ cluster.freeSlots(entry.getValue());
+ if (_isAlive) {
+ _freeSlots.addAll(entry.getValue());
+ }
+ }
+ _topIdToUsedSlots = new HashMap<>();
}
- return total;
- }
-
- public static int countTotalSlotsAlive(Collection<Node> nodes) {
- int total = 0;
- for (Node n: nodes) {
- if (n.isAlive()) {
- total += n.totalSlots();
- }
+
+ /**
+ * Frees a single slot in this node
+ * @param ws the slot to free
+ * @param cluster the cluster to update
+ */
+ public void free(WorkerSlot ws, Cluster cluster, boolean forceFree) {
+ if (_freeSlots.contains(ws)) return;
+ boolean wasFound = false;
+ for (Entry<String, Set<WorkerSlot>> entry : _topIdToUsedSlots.entrySet()) {
+ Set<WorkerSlot> slots = entry.getValue();
+ if (slots.remove(ws)) {
+ cluster.freeSlot(ws);
+ if (_isAlive) {
+ _freeSlots.add(ws);
+ }
+ wasFound = true;
+ }
+ }
+ if (!wasFound) {
+ if (forceFree) {
+ LOG.info("Forcefully freeing the " + ws);
+ cluster.freeSlot(ws);
+ _freeSlots.add(ws);
+ } else {
+ throw new IllegalArgumentException("Tried to free a slot that was not" +
+ " part of this node " + _nodeId);
+ }
+ }
}
- return total;
- }
-
- public static Map<String, Node> getAllNodesFrom(Cluster cluster) {
- Map<String, Node> nodeIdToNode = new HashMap<>();
- for (SupervisorDetails sup : cluster.getSupervisors().values()) {
- //Node ID and supervisor ID are the same.
- String id = sup.getId();
- boolean isAlive = !cluster.isBlackListed(id);
- LOG.debug("Found a {} Node {} {}",
- isAlive? "living":"dead", id, sup.getAllPorts());
- nodeIdToNode.put(id, new Node(id, sup.getAllPorts(), isAlive));
+
+ /**
+ * Frees all the slots for a topology.
+ * @param topId the topology to free slots for
+ * @param cluster the cluster to update
+ */
+ public void freeTopology(String topId, Cluster cluster) {
+ Set<WorkerSlot> slots = _topIdToUsedSlots.get(topId);
+ if (slots == null || slots.isEmpty()) return;
+ for (WorkerSlot ws : slots) {
+ cluster.freeSlot(ws);
+ if (_isAlive) {
+ _freeSlots.add(ws);
+ }
+ }
+ _topIdToUsedSlots.remove(topId);
}
-
- for (Entry<String, SchedulerAssignment> entry : cluster.getAssignments().entrySet()) {
- String topId = entry.getValue().getTopologyId();
- for (WorkerSlot ws: entry.getValue().getSlots()) {
- String id = ws.getNodeId();
- Node node = nodeIdToNode.get(id);
- if (node == null) {
- LOG.debug("Found an assigned slot on a dead supervisor {}", ws);
- node = new Node(id, null, false);
- nodeIdToNode.put(id, node);
+
+ /**
+ * Assign a free slot on the node to the following topology and executors.
+ * This will update the cluster too.
+ * @param topId the topology to assign a free slot to.
+ * @param executors the executors to run in that slot.
+ * @param cluster the cluster to be updated
+ */
+ public void assign(String topId, Collection<ExecutorDetails> executors,
+ Cluster cluster) {
+ if (!_isAlive) {
+ throw new IllegalStateException("Trying to adding to a dead node " + _nodeId);
}
- if (!node.isAlive()) {
- //The supervisor on the node down so add an orphaned slot to hold the unsupervised worker
- node.addOrphanedSlot(ws);
+ if (_freeSlots.isEmpty()) {
+ throw new IllegalStateException("Trying to assign to a full node " + _nodeId);
}
- if (node.assignInternal(ws, topId, true)) {
- LOG.warn("Bad scheduling state for topology [" + topId+ "], the slot " +
- ws + " assigned to multiple workers, un-assigning everything...");
- node.free(ws, cluster, true);
+ if (executors.size() == 0) {
+ LOG.warn("Trying to assign nothing from " + topId + " to " + _nodeId + " (Ignored)");
+ } else {
+ WorkerSlot slot = _freeSlots.iterator().next();
+ cluster.assign(slot, topId, executors);
+ assignInternal(slot, topId, false);
}
- }
}
-
- return nodeIdToNode;
- }
-
- /**
- * Used to sort a list of nodes so the node with the most free slots comes
- * first.
- */
- public static final Comparator<Node> FREE_NODE_COMPARATOR_DEC = new Comparator<Node>() {
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof Node && _nodeId.equals(((Node) other)._nodeId);
+ }
+
+ @Override
+ public int hashCode() {
+ return _nodeId.hashCode();
+ }
+
@Override
- public int compare(Node o1, Node o2) {
- return o1.totalSlotsUsed() - o2.totalSlotsUsed();
+ public String toString() {
+ return "Node: " + _nodeId;
}
- };
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/NodePool.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/NodePool.java b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/NodePool.java
index e609028..21ffbf5 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/NodePool.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/multitenant/NodePool.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.multitenant;
@@ -27,270 +21,268 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.storm.Config;
import org.apache.storm.scheduler.Cluster;
import org.apache.storm.scheduler.ExecutorDetails;
import org.apache.storm.scheduler.SchedulerAssignment;
import org.apache.storm.scheduler.TopologyDetails;
import org.apache.storm.scheduler.WorkerSlot;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A pool of nodes that can be used to run topologies.
*/
public abstract class NodePool {
- protected Cluster _cluster;
- protected Map<String, Node> _nodeIdToNode;
-
- public static class NodeAndSlotCounts {
- public final int _nodes;
- public final int _slots;
-
- public NodeAndSlotCounts(int nodes, int slots) {
- _nodes = nodes;
- _slots = slots;
+ private static final Logger LOG = LoggerFactory.getLogger(NodePool.class);
+ protected Cluster _cluster;
+ protected Map<String, Node> _nodeIdToNode;
+
+ public static int slotsAvailable(NodePool[] pools) {
+ int slotsAvailable = 0;
+ for (NodePool pool : pools) {
+ slotsAvailable += pool.slotsAvailable();
+ }
+ return slotsAvailable;
}
- }
- /**
- * Place executors into slots in a round robin way, taking into account
- * component spreading among different hosts.
- */
- public static class RoundRobinSlotScheduler {
- private Map<String,Set<String>> _nodeToComps;
- private HashMap<String, List<ExecutorDetails>> _spreadToSchedule;
- private LinkedList<Set<ExecutorDetails>> _slots;
- private Set<ExecutorDetails> _lastSlot;
- private Cluster _cluster;
- private String _topId;
-
- /**
- * Create a new scheduler for a given topology
- * @param td the topology to schedule
- * @param slotsToUse the number of slots to use for the executors left to
- * schedule.
- * @param cluster the cluster to schedule this on.
- */
- public RoundRobinSlotScheduler(TopologyDetails td, int slotsToUse,
- Cluster cluster) {
- _topId = td.getId();
- _cluster = cluster;
-
- Map<ExecutorDetails, String> execToComp = td.getExecutorToComponent();
- SchedulerAssignment assignment = _cluster.getAssignmentById(_topId);
- _nodeToComps = new HashMap<>();
+ public static int nodesAvailable(NodePool[] pools) {
+ int nodesAvailable = 0;
+ for (NodePool pool : pools) {
+ nodesAvailable += pool.nodesAvailable();
+ }
+ return nodesAvailable;
+ }
- if (assignment != null) {
- Map<ExecutorDetails, WorkerSlot> execToSlot = assignment.getExecutorToSlot();
-
- for (Entry<ExecutorDetails, WorkerSlot> entry: execToSlot.entrySet()) {
- String nodeId = entry.getValue().getNodeId();
- Set<String> comps = _nodeToComps.get(nodeId);
- if (comps == null) {
- comps = new HashSet<>();
- _nodeToComps.put(nodeId, comps);
- }
- comps.add(execToComp.get(entry.getKey()));
+ public static Collection<Node> takeNodesBySlot(int slotsNeeded, NodePool[] pools) {
+ LOG.debug("Trying to grab {} free slots from {}", slotsNeeded, pools);
+ HashSet<Node> ret = new HashSet<>();
+ for (NodePool pool : pools) {
+ Collection<Node> got = pool.takeNodesBySlots(slotsNeeded);
+ ret.addAll(got);
+ slotsNeeded -= Node.countFreeSlotsAlive(got);
+ LOG.debug("Got {} nodes so far need {} more slots", ret.size(), slotsNeeded);
+ if (slotsNeeded <= 0) {
+ break;
+ }
}
- }
-
- _spreadToSchedule = new HashMap<>();
- List<String> spreadComps = (List<String>)td.getConf().get(Config.TOPOLOGY_SPREAD_COMPONENTS);
- if (spreadComps != null) {
- for (String comp: spreadComps) {
- _spreadToSchedule.put(comp, new ArrayList<ExecutorDetails>());
+ return ret;
+ }
+
+ public static Collection<Node> takeNodes(int nodesNeeded, NodePool[] pools) {
+ LOG.debug("Trying to grab {} free nodes from {}", nodesNeeded, pools);
+ HashSet<Node> ret = new HashSet<>();
+ for (NodePool pool : pools) {
+ Collection<Node> got = pool.takeNodes(nodesNeeded);
+ ret.addAll(got);
+ nodesNeeded -= got.size();
+ LOG.debug("Got {} nodes so far need {} more nodes", ret.size(), nodesNeeded);
+ if (nodesNeeded <= 0) {
+ break;
+ }
}
- }
-
- _slots = new LinkedList<>();
- for (int i = 0; i < slotsToUse; i++) {
- _slots.add(new HashSet<ExecutorDetails>());
- }
+ return ret;
+ }
- int at = 0;
- for (Entry<String, List<ExecutorDetails>> entry: _cluster.getNeedsSchedulingComponentToExecutors(td).entrySet()) {
- LOG.debug("Scheduling for {}", entry.getKey());
- if (_spreadToSchedule.containsKey(entry.getKey())) {
- LOG.debug("Saving {} for spread...",entry.getKey());
- _spreadToSchedule.get(entry.getKey()).addAll(entry.getValue());
- } else {
- for (ExecutorDetails ed: entry.getValue()) {
- LOG.debug("Assigning {} {} to slot {}", entry.getKey(), ed, at);
- _slots.get(at).add(ed);
- at++;
- if (at >= _slots.size()) {
- at = 0;
+ public static int getNodeCountIfSlotsWereTaken(int slots, NodePool[] pools) {
+ LOG.debug("How many nodes to get {} slots from {}", slots, pools);
+ int total = 0;
+ for (NodePool pool : pools) {
+ NodeAndSlotCounts ns = pool.getNodeAndSlotCountIfSlotsWereTaken(slots);
+ total += ns._nodes;
+ slots -= ns._slots;
+ LOG.debug("Found {} nodes so far {} more slots needed", total, slots);
+ if (slots <= 0) {
+ break;
}
- }
}
- }
- _lastSlot = _slots.get(_slots.size() - 1);
+ return total;
}
-
+
/**
- * Assign a slot to the given node.
- * @param n the node to assign a slot to.
- * @return true if there are more slots to assign else false.
+ * Initialize the pool.
+ * @param cluster the cluster
+ * @param nodeIdToNode the mapping of node id to nodes
*/
- public boolean assignSlotTo(Node n) {
- if (_slots.isEmpty()) {
- return false;
- }
- Set<ExecutorDetails> slot = _slots.pop();
- if (slot == _lastSlot) {
- //The last slot fill it up
- for (Entry<String, List<ExecutorDetails>> entry: _spreadToSchedule.entrySet()) {
- if (entry.getValue().size() > 0) {
- slot.addAll(entry.getValue());
- }
- }
- } else {
- String nodeId = n.getId();
- Set<String> nodeComps = _nodeToComps.get(nodeId);
- if (nodeComps == null) {
- nodeComps = new HashSet<>();
- _nodeToComps.put(nodeId, nodeComps);
+ public void init(Cluster cluster, Map<String, Node> nodeIdToNode) {
+ _cluster = cluster;
+ _nodeIdToNode = nodeIdToNode;
+ }
+
+ /**
+ * Add a topology to the pool
+ * @param td the topology to add.
+ */
+ public abstract void addTopology(TopologyDetails td);
+
+ /**
+ * Check if this topology can be added to this pool
+ * @param td the topology
+ * @return true if it can else false
+ */
+ public abstract boolean canAdd(TopologyDetails td);
+
+ /**
+ * @return the number of nodes that are available to be taken
+ */
+ public abstract int slotsAvailable();
+
+ /**
+ * Take nodes from this pool that can fulfill possibly up to the
+ * slotsNeeded
+ * @param slotsNeeded the number of slots that are needed.
+ * @return a Collection of nodes with the removed nodes in it.
+ * This may be empty, but should not be null.
+ */
+ public abstract Collection<Node> takeNodesBySlots(int slotsNeeded);
+
+ /**
+ * Get the number of nodes and slots this would provide to get the slots needed
+ * @param slots the number of slots needed
+ * @return the number of nodes and slots that would be returned.
+ */
+ public abstract NodeAndSlotCounts getNodeAndSlotCountIfSlotsWereTaken(int slots);
+
+ /**
+ * @return the number of nodes that are available to be taken
+ */
+ public abstract int nodesAvailable();
+
+ /**
+ * Take up to nodesNeeded from this pool
+ * @param nodesNeeded the number of nodes that are needed.
+ * @return a Collection of nodes with the removed nodes in it.
+ * This may be empty, but should not be null.
+ */
+ public abstract Collection<Node> takeNodes(int nodesNeeded);
+
+ /**
+ * Reschedule any topologies as needed.
+ * @param lesserPools pools that may be used to steal nodes from.
+ */
+ public abstract void scheduleAsNeeded(NodePool... lesserPools);
+
+ public static class NodeAndSlotCounts {
+ public final int _nodes;
+ public final int _slots;
+
+ public NodeAndSlotCounts(int nodes, int slots) {
+ _nodes = nodes;
+ _slots = slots;
}
- for (Entry<String, List<ExecutorDetails>> entry: _spreadToSchedule.entrySet()) {
- if (entry.getValue().size() > 0) {
- String comp = entry.getKey();
- if (!nodeComps.contains(comp)) {
- nodeComps.add(comp);
- slot.add(entry.getValue().remove(0));
+ }
+
+ /**
+ * Place executors into slots in a round robin way, taking into account
+ * component spreading among different hosts.
+ */
+ public static class RoundRobinSlotScheduler {
+ private Map<String, Set<String>> _nodeToComps;
+ private HashMap<String, List<ExecutorDetails>> _spreadToSchedule;
+ private LinkedList<Set<ExecutorDetails>> _slots;
+ private Set<ExecutorDetails> _lastSlot;
+ private Cluster _cluster;
+ private String _topId;
+
+ /**
+ * Create a new scheduler for a given topology
+ * @param td the topology to schedule
+ * @param slotsToUse the number of slots to use for the executors left to
+ * schedule.
+ * @param cluster the cluster to schedule this on.
+ */
+ public RoundRobinSlotScheduler(TopologyDetails td, int slotsToUse,
+ Cluster cluster) {
+ _topId = td.getId();
+ _cluster = cluster;
+
+ Map<ExecutorDetails, String> execToComp = td.getExecutorToComponent();
+ SchedulerAssignment assignment = _cluster.getAssignmentById(_topId);
+ _nodeToComps = new HashMap<>();
+
+ if (assignment != null) {
+ Map<ExecutorDetails, WorkerSlot> execToSlot = assignment.getExecutorToSlot();
+
+ for (Entry<ExecutorDetails, WorkerSlot> entry : execToSlot.entrySet()) {
+ String nodeId = entry.getValue().getNodeId();
+ Set<String> comps = _nodeToComps.get(nodeId);
+ if (comps == null) {
+ comps = new HashSet<>();
+ _nodeToComps.put(nodeId, comps);
+ }
+ comps.add(execToComp.get(entry.getKey()));
+ }
+ }
+
+ _spreadToSchedule = new HashMap<>();
+ List<String> spreadComps = (List<String>) td.getConf().get(Config.TOPOLOGY_SPREAD_COMPONENTS);
+ if (spreadComps != null) {
+ for (String comp : spreadComps) {
+ _spreadToSchedule.put(comp, new ArrayList<ExecutorDetails>());
+ }
+ }
+
+ _slots = new LinkedList<>();
+ for (int i = 0; i < slotsToUse; i++) {
+ _slots.add(new HashSet<ExecutorDetails>());
+ }
+
+ int at = 0;
+ for (Entry<String, List<ExecutorDetails>> entry : _cluster.getNeedsSchedulingComponentToExecutors(td).entrySet()) {
+ LOG.debug("Scheduling for {}", entry.getKey());
+ if (_spreadToSchedule.containsKey(entry.getKey())) {
+ LOG.debug("Saving {} for spread...", entry.getKey());
+ _spreadToSchedule.get(entry.getKey()).addAll(entry.getValue());
+ } else {
+ for (ExecutorDetails ed : entry.getValue()) {
+ LOG.debug("Assigning {} {} to slot {}", entry.getKey(), ed, at);
+ _slots.get(at).add(ed);
+ at++;
+ if (at >= _slots.size()) {
+ at = 0;
+ }
+ }
+ }
}
- }
+ _lastSlot = _slots.get(_slots.size() - 1);
}
- }
- n.assign(_topId, slot, _cluster);
- return !_slots.isEmpty();
- }
- }
-
- private static final Logger LOG = LoggerFactory.getLogger(NodePool.class);
- /**
- * Initialize the pool.
- * @param cluster the cluster
- * @param nodeIdToNode the mapping of node id to nodes
- */
- public void init(Cluster cluster, Map<String, Node> nodeIdToNode) {
- _cluster = cluster;
- _nodeIdToNode = nodeIdToNode;
- }
-
- /**
- * Add a topology to the pool
- * @param td the topology to add.
- */
- public abstract void addTopology(TopologyDetails td);
-
- /**
- * Check if this topology can be added to this pool
- * @param td the topology
- * @return true if it can else false
- */
- public abstract boolean canAdd(TopologyDetails td);
-
- /**
- * @return the number of nodes that are available to be taken
- */
- public abstract int slotsAvailable();
-
- /**
- * Take nodes from this pool that can fulfill possibly up to the
- * slotsNeeded
- * @param slotsNeeded the number of slots that are needed.
- * @return a Collection of nodes with the removed nodes in it.
- * This may be empty, but should not be null.
- */
- public abstract Collection<Node> takeNodesBySlots(int slotsNeeded);
- /**
- * Get the number of nodes and slots this would provide to get the slots needed
- * @param slots the number of slots needed
- * @return the number of nodes and slots that would be returned.
- */
- public abstract NodeAndSlotCounts getNodeAndSlotCountIfSlotsWereTaken(int slots);
-
- /**
- * @return the number of nodes that are available to be taken
- */
- public abstract int nodesAvailable();
-
- /**
- * Take up to nodesNeeded from this pool
- * @param nodesNeeded the number of nodes that are needed.
- * @return a Collection of nodes with the removed nodes in it.
- * This may be empty, but should not be null.
- */
- public abstract Collection<Node> takeNodes(int nodesNeeded);
-
- /**
- * Reschedule any topologies as needed.
- * @param lesserPools pools that may be used to steal nodes from.
- */
- public abstract void scheduleAsNeeded(NodePool ... lesserPools);
-
- public static int slotsAvailable(NodePool[] pools) {
- int slotsAvailable = 0;
- for (NodePool pool: pools) {
- slotsAvailable += pool.slotsAvailable();
- }
- return slotsAvailable;
- }
-
- public static int nodesAvailable(NodePool[] pools) {
- int nodesAvailable = 0;
- for (NodePool pool: pools) {
- nodesAvailable += pool.nodesAvailable();
- }
- return nodesAvailable;
- }
-
- public static Collection<Node> takeNodesBySlot(int slotsNeeded,NodePool[] pools) {
- LOG.debug("Trying to grab {} free slots from {}",slotsNeeded, pools);
- HashSet<Node> ret = new HashSet<>();
- for (NodePool pool: pools) {
- Collection<Node> got = pool.takeNodesBySlots(slotsNeeded);
- ret.addAll(got);
- slotsNeeded -= Node.countFreeSlotsAlive(got);
- LOG.debug("Got {} nodes so far need {} more slots",ret.size(),slotsNeeded);
- if (slotsNeeded <= 0) {
- break;
- }
- }
- return ret;
- }
-
- public static Collection<Node> takeNodes(int nodesNeeded,NodePool[] pools) {
- LOG.debug("Trying to grab {} free nodes from {}",nodesNeeded, pools);
- HashSet<Node> ret = new HashSet<>();
- for (NodePool pool: pools) {
- Collection<Node> got = pool.takeNodes(nodesNeeded);
- ret.addAll(got);
- nodesNeeded -= got.size();
- LOG.debug("Got {} nodes so far need {} more nodes", ret.size(), nodesNeeded);
- if (nodesNeeded <= 0) {
- break;
- }
+ /**
+ * Assign a slot to the given node.
+ * @param n the node to assign a slot to.
+ * @return true if there are more slots to assign else false.
+ */
+ public boolean assignSlotTo(Node n) {
+ if (_slots.isEmpty()) {
+ return false;
+ }
+ Set<ExecutorDetails> slot = _slots.pop();
+ if (slot == _lastSlot) {
+ //The last slot fill it up
+ for (Entry<String, List<ExecutorDetails>> entry : _spreadToSchedule.entrySet()) {
+ if (entry.getValue().size() > 0) {
+ slot.addAll(entry.getValue());
+ }
+ }
+ } else {
+ String nodeId = n.getId();
+ Set<String> nodeComps = _nodeToComps.get(nodeId);
+ if (nodeComps == null) {
+ nodeComps = new HashSet<>();
+ _nodeToComps.put(nodeId, nodeComps);
+ }
+ for (Entry<String, List<ExecutorDetails>> entry : _spreadToSchedule.entrySet()) {
+ if (entry.getValue().size() > 0) {
+ String comp = entry.getKey();
+ if (!nodeComps.contains(comp)) {
+ nodeComps.add(comp);
+ slot.add(entry.getValue().remove(0));
+ }
+ }
+ }
+ }
+ n.assign(_topId, slot, _cluster);
+ return !_slots.isEmpty();
+ }
}
- return ret;
- }
-
- public static int getNodeCountIfSlotsWereTaken(int slots,NodePool[] pools) {
- LOG.debug("How many nodes to get {} slots from {}",slots, pools);
- int total = 0;
- for (NodePool pool: pools) {
- NodeAndSlotCounts ns = pool.getNodeAndSlotCountIfSlotsWereTaken(slots);
- total += ns._nodes;
- slots -= ns._slots;
- LOG.debug("Found {} nodes so far {} more slots needed", total, slots);
- if (slots <= 0) {
- break;
- }
- }
- return total;
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/RAS_Node.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/RAS_Node.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/RAS_Node.java
index 84a95a7..4f43c0a 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/RAS_Node.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/RAS_Node.java
@@ -34,24 +34,23 @@ import org.apache.storm.scheduler.resource.normalization.NormalizedResourceOffer
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/** Represents a single node in the cluster. */
+/**
+ * Represents a single node in the cluster.
+ */
public class RAS_Node {
private static final Logger LOG = LoggerFactory.getLogger(RAS_Node.class);
-
+ private final String nodeId;
+ private final Cluster cluster;
+ private final Set<WorkerSlot> originallyFreeSlots;
//A map consisting of all workers on the node.
//The key of the map is the worker id and the value is the corresponding workerslot object
private Map<String, WorkerSlot> slots = new HashMap<>();
-
// A map describing which topologies are using which slots on this node. The format of the map is the following:
// {TopologyId -> {WorkerId -> {Executors}}}
private Map<String, Map<String, Collection<ExecutorDetails>>> topIdToUsedSlots = new HashMap<>();
-
- private final String nodeId;
private String hostname;
private boolean isAlive;
private SupervisorDetails sup;
- private final Cluster cluster;
- private final Set<WorkerSlot> originallyFreeSlots;
public RAS_Node(
String nodeId,
@@ -99,6 +98,26 @@ public class RAS_Node {
}
}
+ public static int countFreeSlotsAlive(Collection<RAS_Node> nodes) {
+ int total = 0;
+ for (RAS_Node n : nodes) {
+ if (n.isAlive()) {
+ total += n.totalSlotsFree();
+ }
+ }
+ return total;
+ }
+
+ public static int countTotalSlotsAlive(Collection<RAS_Node> nodes) {
+ int total = 0;
+ for (RAS_Node n : nodes) {
+ if (n.isAlive()) {
+ total += n.totalSlots();
+ }
+ }
+ return total;
+ }
+
public String getId() {
return nodeId;
}
@@ -167,7 +186,9 @@ public class RAS_Node {
return isAlive;
}
- /** Get a collection of the topology ids currently running on this node. */
+ /**
+ * Get a collection of the topology ids currently running on this node.
+ */
public Collection<String> getRunningTopologies() {
return topIdToUsedSlots.keySet();
}
@@ -192,7 +213,9 @@ public class RAS_Node {
return slots.size();
}
- /** Free all slots on this node. This will update the Cluster too. */
+ /**
+ * Free all slots on this node. This will update the Cluster too.
+ */
public void freeAllSlots() {
if (!isAlive) {
LOG.warn("Freeing all slots on a dead node {} ", nodeId);
@@ -279,8 +302,8 @@ public class RAS_Node {
/**
* Assigns a worker to a node.
*
- * @param target the worker slot to assign the executors
- * @param td the topology the executors are from
+ * @param target the worker slot to assign the executors
+ * @param td the topology the executors are from
* @param executors executors to assign to the specified worker slot
*/
public void assign(WorkerSlot target, TopologyDetails td, Collection<ExecutorDetails> executors) {
@@ -343,9 +366,9 @@ public class RAS_Node {
/**
* Would scheduling exec in ws fit with the current resource constraints.
*
- * @param ws the slot to possibly put exec in
+ * @param ws the slot to possibly put exec in
* @param exec the executor to possibly place in ws
- * @param td the topology exec is a part of
+ * @param td the topology exec is a part of
* @return true if it would fit else false
*/
public boolean wouldFit(WorkerSlot ws, ExecutorDetails exec, TopologyDetails td) {
@@ -353,13 +376,13 @@ public class RAS_Node {
throw new IllegalStateException("Slot " + ws + " is not a part of this node " + nodeId);
}
return isAlive
- && cluster.wouldFit(
+ && cluster.wouldFit(
ws,
exec,
td,
getTotalAvailableResources(),
td.getTopologyWorkerMaxHeapSize()
- );
+ );
}
@Override
@@ -378,40 +401,20 @@ public class RAS_Node {
@Override
public String toString() {
return "{Node: "
- + ((sup == null) ? "null (possibly down)" : sup.getHost())
- + ", Avail [ Mem: "
- + getAvailableMemoryResources()
- + ", CPU: "
- + getAvailableCpuResources()
- + ", Slots: "
- + this.getFreeSlots()
- + "] Total [ Mem: "
- + ((sup == null) ? "N/A" : this.getTotalMemoryResources())
- + ", CPU: "
- + ((sup == null) ? "N/A" : this.getTotalCpuResources())
- + ", Slots: "
- + this.slots.values()
- + " ]}";
- }
-
- public static int countFreeSlotsAlive(Collection<RAS_Node> nodes) {
- int total = 0;
- for (RAS_Node n : nodes) {
- if (n.isAlive()) {
- total += n.totalSlotsFree();
- }
- }
- return total;
- }
-
- public static int countTotalSlotsAlive(Collection<RAS_Node> nodes) {
- int total = 0;
- for (RAS_Node n : nodes) {
- if (n.isAlive()) {
- total += n.totalSlots();
- }
- }
- return total;
+ + ((sup == null) ? "null (possibly down)" : sup.getHost())
+ + ", Avail [ Mem: "
+ + getAvailableMemoryResources()
+ + ", CPU: "
+ + getAvailableCpuResources()
+ + ", Slots: "
+ + this.getFreeSlots()
+ + "] Total [ Mem: "
+ + ((sup == null) ? "N/A" : this.getTotalMemoryResources())
+ + ", CPU: "
+ + ((sup == null) ? "N/A" : this.getTotalCpuResources())
+ + ", Slots: "
+ + this.slots.values()
+ + " ]}";
}
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/RAS_Nodes.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/RAS_Nodes.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/RAS_Nodes.java
index 2bd2b86..584934b 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/RAS_Nodes.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/RAS_Nodes.java
@@ -32,9 +32,8 @@ import org.slf4j.LoggerFactory;
public class RAS_Nodes {
- private Map<String, RAS_Node> nodeMap;
-
private static final Logger LOG = LoggerFactory.getLogger(RAS_Nodes.class);
+ private Map<String, RAS_Node> nodeMap;
public RAS_Nodes(Cluster cluster) {
this.nodeMap = getAllNodesFrom(cluster);
@@ -116,13 +115,16 @@ public class RAS_Nodes {
return nodeIdToNode;
}
- /** get node object from nodeId. */
+ /**
+ * get node object from nodeId.
+ */
public RAS_Node getNodeById(String nodeId) {
return this.nodeMap.get(nodeId);
}
/**
* Free everything on the given slots.
+ *
* @param workerSlots the slots to free
*/
public void freeSlots(Collection<WorkerSlot> workerSlots) {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceAwareScheduler.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceAwareScheduler.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceAwareScheduler.java
index 541e1a5..60f6f6c 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceAwareScheduler.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceAwareScheduler.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource;
@@ -52,11 +46,35 @@ public class ResourceAwareScheduler implements IScheduler {
private IConfigLoader configLoader;
private int maxSchedulingAttempts;
+ private static void markFailedTopology(User u, Cluster c, TopologyDetails td, String message) {
+ markFailedTopology(u, c, td, message, null);
+ }
+
+ private static void markFailedTopology(User u, Cluster c, TopologyDetails td, String message, Throwable t) {
+ c.setStatus(td, message);
+ String realMessage = td.getId() + " " + message;
+ if (t != null) {
+ LOG.error(realMessage, t);
+ } else {
+ LOG.error(realMessage);
+ }
+ u.markTopoUnsuccess(td);
+ }
+
+ private static double getCpuUsed(SchedulerAssignment assignment) {
+ return assignment.getScheduledResources().values().stream().mapToDouble((wr) -> wr.get_cpu()).sum();
+ }
+
+ private static double getMemoryUsed(SchedulerAssignment assignment) {
+ return assignment.getScheduledResources().values().stream()
+ .mapToDouble((wr) -> wr.get_mem_on_heap() + wr.get_mem_off_heap()).sum();
+ }
+
@Override
public void prepare(Map<String, Object> conf) {
this.conf = conf;
schedulingPriorityStrategy = ReflectionUtils.newInstance(
- (String) conf.get(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY));
+ (String) conf.get(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY));
configLoader = ConfigLoaderFactoryService.createConfigLoader(conf);
maxSchedulingAttempts = ObjectReader.getInt(
conf.get(DaemonConfig.RESOURCE_AWARE_SCHEDULER_MAX_TOPOLOGY_SCHEDULING_ATTEMPTS), 5);
@@ -85,21 +103,6 @@ public class ResourceAwareScheduler implements IScheduler {
}
}
- private static void markFailedTopology(User u, Cluster c, TopologyDetails td, String message) {
- markFailedTopology(u, c, td, message, null);
- }
-
- private static void markFailedTopology(User u, Cluster c, TopologyDetails td, String message, Throwable t) {
- c.setStatus(td, message);
- String realMessage = td.getId() + " " + message;
- if (t != null) {
- LOG.error(realMessage, t);
- } else {
- LOG.error(realMessage);
- }
- u.markTopoUnsuccess(td);
- }
-
private void scheduleTopology(TopologyDetails td, Cluster cluster, final User topologySubmitter,
List<TopologyDetails> orderedTopologies) {
//A copy of cluster that we can modify, but does not get committed back to cluster unless scheduling succeeds
@@ -119,17 +122,17 @@ public class ResourceAwareScheduler implements IScheduler {
rasStrategy.prepare(conf);
} catch (DisallowedStrategyException e) {
markFailedTopology(topologySubmitter, cluster, td,
- "Unsuccessful in scheduling - " + e.getAttemptedClass()
- + " is not an allowed strategy. Please make sure your "
- + Config.TOPOLOGY_SCHEDULER_STRATEGY
- + " config is one of the allowed strategies: "
- + e.getAllowedStrategies(), e);
+ "Unsuccessful in scheduling - " + e.getAttemptedClass()
+ + " is not an allowed strategy. Please make sure your "
+ + Config.TOPOLOGY_SCHEDULER_STRATEGY
+ + " config is one of the allowed strategies: "
+ + e.getAllowedStrategies(), e);
return;
} catch (RuntimeException e) {
markFailedTopology(topologySubmitter, cluster, td,
- "Unsuccessful in scheduling - failed to create instance of topology strategy "
- + strategyConf
- + ". Please check logs for details", e);
+ "Unsuccessful in scheduling - failed to create instance of topology strategy "
+ + strategyConf
+ + ". Please check logs for details", e);
return;
}
@@ -167,7 +170,7 @@ public class ResourceAwareScheduler implements IScheduler {
Collection<WorkerSlot> workersToEvict = workingState.getUsedSlotsByTopologyId(topologyEvict.getId());
LOG.debug("Evicting Topology {} with workers: {} from user {}", topologyEvict.getName(), workersToEvict,
- topologyEvict.getTopologySubmitter());
+ topologyEvict.getTopologySubmitter());
cpuNeeded -= getCpuUsed(evictAssignemnt);
memoryNeeded -= getMemoryUsed(evictAssignemnt);
evictedSomething = true;
@@ -205,22 +208,13 @@ public class ResourceAwareScheduler implements IScheduler {
}
} catch (Exception ex) {
markFailedTopology(topologySubmitter, cluster, td,
- "Internal Error - Exception thrown when scheduling. Please check logs for details", ex);
+ "Internal Error - Exception thrown when scheduling. Please check logs for details", ex);
return;
}
}
markFailedTopology(topologySubmitter, cluster, td, "Failed to schedule within " + maxSchedulingAttempts + " attempts");
}
- private static double getCpuUsed(SchedulerAssignment assignment) {
- return assignment.getScheduledResources().values().stream().mapToDouble((wr) -> wr.get_cpu()).sum();
- }
-
- private static double getMemoryUsed(SchedulerAssignment assignment) {
- return assignment.getScheduledResources().values().stream()
- .mapToDouble((wr) -> wr.get_mem_on_heap() + wr.get_mem_off_heap()).sum();
- }
-
/**
* Get User wrappers around cluster.
*
@@ -288,7 +282,7 @@ public class ResourceAwareScheduler implements IScheduler {
return convertToDouble(raw);
} else {
LOG.warn("Reading from user-resource-pools.yaml returned null. This could because the file is not available. "
- + "Will load configs from storm configuration");
+ + "Will load configs from storm configuration");
}
// if no configs from user-resource-pools.yaml, get configs from conf
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
index 2f9ab4c..e0a8114 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource;
@@ -37,7 +31,7 @@ public class ResourceUtils {
private static final Logger LOG = LoggerFactory.getLogger(ResourceUtils.class);
public static NormalizedResourceRequest getBoltResources(StormTopology topology, Map<String, Object> topologyConf,
- String componentId) {
+ String componentId) {
if (topology.get_bolts() != null) {
Bolt bolt = topology.get_bolts().get(componentId);
return new NormalizedResourceRequest(bolt.get_common(), topologyConf);
@@ -46,7 +40,7 @@ public class ResourceUtils {
}
public static Map<String, NormalizedResourceRequest> getBoltsResources(StormTopology topology,
- Map<String, Object> topologyConf) {
+ Map<String, Object> topologyConf) {
Map<String, NormalizedResourceRequest> boltResources = new HashMap<>();
if (topology.get_bolts() != null) {
for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) {
@@ -70,7 +64,7 @@ public class ResourceUtils {
}
public static Map<String, NormalizedResourceRequest> getSpoutsResources(StormTopology topology,
- Map<String, Object> topologyConf) {
+ Map<String, Object> topologyConf) {
Map<String, NormalizedResourceRequest> spoutResources = new HashMap<>();
if (topology.get_spouts() != null) {
for (Map.Entry<String, SpoutSpec> spout : topology.get_spouts().entrySet()) {
@@ -128,7 +122,7 @@ public class ResourceUtils {
}
public static String getCorrespondingLegacyResourceName(String normalizedResourceName) {
- for(Map.Entry<String, String> entry : NormalizedResources.RESOURCE_NAME_NORMALIZER.getResourceNameMapping().entrySet()) {
+ for (Map.Entry<String, String> entry : NormalizedResources.RESOURCE_NAME_NORMALIZER.getResourceNameMapping().entrySet()) {
if (entry.getValue().equals(normalizedResourceName)) {
return entry.getKey();
}
@@ -143,9 +137,9 @@ public class ResourceUtils {
JSONObject jsonObject = (JSONObject) obj;
Map<String, Double> componentResourceMap =
- (Map<String, Double>) jsonObject.getOrDefault(
- Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap<String, Double>()
- );
+ (Map<String, Double>) jsonObject.getOrDefault(
+ Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap<String, Double>()
+ );
for (Map.Entry<String, Double> resourceUpdateEntry : resourceUpdates.entrySet()) {
if (NormalizedResources.RESOURCE_NAME_NORMALIZER.getResourceNameMapping().containsValue(resourceUpdateEntry.getKey())) {
@@ -159,7 +153,7 @@ public class ResourceUtils {
return jsonObject.toJSONString();
} catch (ParseException ex) {
- throw new RuntimeException("Failed to parse component resources with json: " + jsonConf);
+ throw new RuntimeException("Failed to parse component resources with json: " + jsonConf);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingResult.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingResult.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingResult.java
index 72f083d..882c6be 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingResult.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingResult.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource;
@@ -66,7 +60,7 @@ public class SchedulingResult {
public String getErrorMessage() {
return this.errorMessage;
}
-
+
public boolean isSuccess() {
return SchedulingStatus.isStatusSuccess(this.status);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingStatus.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingStatus.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingStatus.java
index 703f4b8..47c0541 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingStatus.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/SchedulingStatus.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource;
@@ -28,7 +22,7 @@ public enum SchedulingStatus {
public static EnumSet<SchedulingStatus> success = EnumSet.of(SUCCESS);
public static EnumSet<SchedulingStatus> failure = EnumSet.of(FAIL_INVALID_TOPOLOGY, FAIL_NOT_ENOUGH_RESOURCES,
- FAIL_OTHER);
+ FAIL_OTHER);
public static boolean isStatusSuccess(SchedulingStatus status) {
return success.contains(status);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/User.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/User.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/User.java
index 9083510..ee79d59 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/User.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/User.java
@@ -18,13 +18,11 @@
package org.apache.storm.scheduler.resource;
-import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-
import org.apache.storm.daemon.nimbus.TopologyResources;
import org.apache.storm.scheduler.Cluster;
import org.apache.storm.scheduler.ISchedulingState;
@@ -32,13 +30,11 @@ import org.apache.storm.scheduler.SchedulerAssignment;
import org.apache.storm.scheduler.TopologyDetails;
public class User {
- private String userId;
-
//Topologies that were deemed to be invalid
private final Set<TopologyDetails> unsuccess = new HashSet<>();
-
private final double cpuGuarantee;
private final double memoryGuarantee;
+ private String userId;
public User(String userId) {
this(userId, 0, 0);
@@ -206,8 +202,8 @@ public class User {
}
/**
- * Comparator that sorts topologies by priority and then by submission time First sort by Topology
- * Priority, if there is a tie for topology priority, topology uptime is used to sort.
+ * Comparator that sorts topologies by priority and then by submission time First sort by Topology Priority, if there is a tie for
+ * topology priority, topology uptime is used to sort.
*/
static class PQsortByPriorityAndSubmittionTime implements Comparator<TopologyDetails> {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOffer.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOffer.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOffer.java
index 417dca9..0c073a8 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOffer.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceOffer.java
@@ -74,12 +74,13 @@ public class NormalizedResourceOffer implements NormalizedResourcesWithMemory {
if (totalMemoryMb < 0.0) {
normalizedResources.throwBecauseResourceBecameNegative(
Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME, totalMemoryMb, other.getTotalMemoryMb());
- };
+ }
+ ;
}
/**
* @see NormalizedResources#calculateAveragePercentageUsedBy(org.apache.storm.scheduler.resource.normalization.NormalizedResources,
- * double, double).
+ * double, double).
*/
public double calculateAveragePercentageUsedBy(NormalizedResourceOffer used) {
return normalizedResources.calculateAveragePercentageUsedBy(
@@ -88,7 +89,7 @@ public class NormalizedResourceOffer implements NormalizedResourcesWithMemory {
/**
* @see NormalizedResources#calculateMinPercentageUsedBy(org.apache.storm.scheduler.resource.normalization.NormalizedResources, double,
- * double)
+ * double)
*/
public double calculateMinPercentageUsedBy(NormalizedResourceOffer used) {
return normalizedResources.calculateMinPercentageUsedBy(used.getNormalizedResources(), getTotalMemoryMb(), used.getTotalMemoryMb());
@@ -96,7 +97,7 @@ public class NormalizedResourceOffer implements NormalizedResourcesWithMemory {
/**
* @see NormalizedResources#couldHoldIgnoringSharedMemory(org.apache.storm.scheduler.resource.normalization.NormalizedResources, double,
- * double).
+ * double).
*/
public boolean couldHoldIgnoringSharedMemory(NormalizedResourcesWithMemory other) {
return normalizedResources.couldHoldIgnoringSharedMemory(
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequest.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequest.java
index 6627bb5..dc380e6 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequest.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResourceRequest.java
@@ -37,6 +37,28 @@ import org.slf4j.LoggerFactory;
public class NormalizedResourceRequest implements NormalizedResourcesWithMemory {
private static final Logger LOG = LoggerFactory.getLogger(NormalizedResourceRequest.class);
+ private final NormalizedResources normalizedResources;
+ private double onHeap;
+ private double offHeap;
+
+ private NormalizedResourceRequest(Map<String, ? extends Number> resources,
+ Map<String, Double> defaultResources) {
+ Map<String, Double> normalizedResourceMap = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(defaultResources);
+ normalizedResourceMap.putAll(NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resources));
+ onHeap = normalizedResourceMap.getOrDefault(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, 0.0);
+ offHeap = normalizedResourceMap.getOrDefault(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, 0.0);
+ normalizedResources = new NormalizedResources(normalizedResourceMap);
+ }
+ public NormalizedResourceRequest(ComponentCommon component, Map<String, Object> topoConf) {
+ this(parseResources(component.get_json_conf()), getDefaultResources(topoConf));
+ }
+ public NormalizedResourceRequest(Map<String, Object> topoConf) {
+ this((Map<String, ? extends Number>) null, getDefaultResources(topoConf));
+ }
+
+ public NormalizedResourceRequest() {
+ this((Map<String, ? extends Number>) null, null);
+ }
private static void putIfMissing(Map<String, Double> dest, String destKey, Map<String, Object> src, String srcKey) {
if (!dest.containsKey(destKey)) {
@@ -48,8 +70,9 @@ public class NormalizedResourceRequest implements NormalizedResourcesWithMemory
}
private static Map<String, Double> getDefaultResources(Map<String, Object> topoConf) {
- Map<String, Double> ret = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap((Map<String, Number>) topoConf.getOrDefault(
- Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap<>()));
+ Map<String, Double> ret =
+ NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap((Map<String, Number>) topoConf.getOrDefault(
+ Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, new HashMap<>()));
putIfMissing(ret, Constants.COMMON_CPU_RESOURCE_NAME, topoConf, Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT);
putIfMissing(ret, Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, topoConf, Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB);
putIfMissing(ret, Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, topoConf, Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB);
@@ -78,7 +101,7 @@ public class NormalizedResourceRequest implements NormalizedResourcesWithMemory
}
if (jsonObject.containsKey(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)) {
Double topoCpu = ObjectReader.getDouble(jsonObject.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT),
- null);
+ null);
topologyResources.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, topoCpu);
}
@@ -102,31 +125,6 @@ public class NormalizedResourceRequest implements NormalizedResourcesWithMemory
return topologyResources;
}
- private final NormalizedResources normalizedResources;
- private double onHeap;
- private double offHeap;
-
- private NormalizedResourceRequest(Map<String, ? extends Number> resources,
- Map<String, Double> defaultResources) {
- Map<String, Double> normalizedResourceMap = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(defaultResources);
- normalizedResourceMap.putAll(NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resources));
- onHeap = normalizedResourceMap.getOrDefault(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME, 0.0);
- offHeap = normalizedResourceMap.getOrDefault(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, 0.0);
- normalizedResources = new NormalizedResources(normalizedResourceMap);
- }
-
- public NormalizedResourceRequest(ComponentCommon component, Map<String, Object> topoConf) {
- this(parseResources(component.get_json_conf()), getDefaultResources(topoConf));
- }
-
- public NormalizedResourceRequest(Map<String, Object> topoConf) {
- this((Map<String, ? extends Number>) null, getDefaultResources(topoConf));
- }
-
- public NormalizedResourceRequest() {
- this((Map<String, ? extends Number>) null, null);
- }
-
public Map<String, Double> toNormalizedMap() {
Map<String, Double> ret = this.normalizedResources.toNormalizedMap();
ret.put(Constants.COMMON_OFFHEAP_MEMORY_RESOURCE_NAME, offHeap);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResources.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResources.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResources.java
index 76d5ce2..c41cd95 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResources.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/NormalizedResources.java
@@ -45,16 +45,6 @@ public class NormalizedResources {
private double[] otherResources;
/**
- * This is for testing only. It allows a test to reset the static state relating to resource names. We reset the mapping because some
- * algorithms sadly have different behavior if a resource exists or not.
- */
- @VisibleForTesting
- public static void resetResourceNames() {
- RESOURCE_NAME_NORMALIZER = new ResourceNameNormalizer();
- RESOURCE_MAP_ARRAY_BRIDGE = new ResourceMapArrayBridge();
- }
-
- /**
* Copy constructor.
*/
public NormalizedResources(NormalizedResources other) {
@@ -74,6 +64,16 @@ public class NormalizedResources {
}
/**
+ * This is for testing only. It allows a test to reset the static state relating to resource names. We reset the mapping because some
+ * algorithms sadly have different behavior if a resource exists or not.
+ */
+ @VisibleForTesting
+ public static void resetResourceNames() {
+ RESOURCE_NAME_NORMALIZER = new ResourceNameNormalizer();
+ RESOURCE_MAP_ARRAY_BRIDGE = new ResourceMapArrayBridge();
+ }
+
+ /**
* Get the total amount of cpu.
*
* @return the amount of cpu.
@@ -81,7 +81,7 @@ public class NormalizedResources {
public double getTotalCpu() {
return cpu;
}
-
+
private void zeroPadOtherResourcesIfNecessary(int requiredLength) {
if (requiredLength > otherResources.length) {
double[] newResources = new double[requiredLength];
@@ -116,16 +116,18 @@ public class NormalizedResources {
/**
* Throw an IllegalArgumentException because a resource became negative during remove.
- * @param resourceName The name of the resource that became negative
- * @param currentValue The current value of the resource
+ *
+ * @param resourceName The name of the resource that became negative
+ * @param currentValue The current value of the resource
* @param subtractedValue The value that was subtracted to make the resource negative
*/
public void throwBecauseResourceBecameNegative(String resourceName, double currentValue, double subtractedValue) {
throw new IllegalArgumentException(String.format("Resource amounts should never be negative."
- + " Resource '%s' with current value '%f' became negative because '%f' was removed.",
- resourceName, currentValue, subtractedValue));
+ +
+ " Resource '%s' with current value '%f' became negative because '%f' was removed.",
+ resourceName, currentValue, subtractedValue));
}
-
+
/**
* Remove the other resources from this. This is the same as subtracting the resources in other from this.
*
@@ -173,8 +175,8 @@ public class NormalizedResources {
* A simple sanity check to see if all of the resources in this would be large enough to hold the resources in other ignoring memory. It
* does not check memory because with shared memory it is beyond the scope of this.
*
- * @param other the resources that we want to check if they would fit in this.
- * @param thisTotalMemoryMb The total memory in MB of this
+ * @param other the resources that we want to check if they would fit in this.
+ * @param thisTotalMemoryMb The total memory in MB of this
* @param otherTotalMemoryMb The total memory in MB of other
* @return true if it might fit, else false if it could not possibly fit.
*/
@@ -204,8 +206,8 @@ public class NormalizedResources {
private void throwBecauseUsedIsNotSubsetOfTotal(NormalizedResources used, double totalMemoryMb, double usedMemoryMb) {
throw new IllegalArgumentException(String.format("The used resources must be a subset of the total resources."
- + " Used: '%s', Total: '%s', Used Mem: '%f', Total Mem: '%f'",
- used.toNormalizedMap(), this.toNormalizedMap(), usedMemoryMb, totalMemoryMb));
+ + " Used: '%s', Total: '%s', Used Mem: '%f', Total Mem: '%f'",
+ used.toNormalizedMap(), this.toNormalizedMap(), usedMemoryMb, totalMemoryMb));
}
/**
@@ -213,18 +215,19 @@ public class NormalizedResources {
* subset of the total resources. If a resource in the total has a value of zero, it will be skipped in the calculation to avoid
* division by 0. If all resources are skipped the result is defined to be 100.0.
*
- * @param used the amount of resources used.
+ * @param used the amount of resources used.
* @param totalMemoryMb The total memory in MB
- * @param usedMemoryMb The used memory in MB
+ * @param usedMemoryMb The used memory in MB
* @return the average percentage used 0.0 to 100.0.
+ *
* @throws IllegalArgumentException if any resource in used has a greater value than the same resource in the total, or used has generic
- * resources that are not present in the total.
+ * resources that are not present in the total.
*/
public double calculateAveragePercentageUsedBy(NormalizedResources used, double totalMemoryMb, double usedMemoryMb) {
if (LOG.isTraceEnabled()) {
LOG.trace("Calculating avg percentage used by. Used Mem: {} Total Mem: {}"
- + " Used Normalized Resources: {} Total Normalized Resources: {}", totalMemoryMb, usedMemoryMb,
- toNormalizedMap(), used.toNormalizedMap());
+ + " Used Normalized Resources: {} Total Normalized Resources: {}", totalMemoryMb, usedMemoryMb,
+ toNormalizedMap(), used.toNormalizedMap());
}
int skippedResourceTypes = 0;
@@ -290,18 +293,19 @@ public class NormalizedResources {
* subset of the total resources. If a resource in the total has a value of zero, it will be skipped in the calculation to avoid
* division by 0. If all resources are skipped the result is defined to be 100.0.
*
- * @param used the amount of resources used.
+ * @param used the amount of resources used.
* @param totalMemoryMb The total memory in MB
- * @param usedMemoryMb The used memory in MB
+ * @param usedMemoryMb The used memory in MB
* @return the minimum percentage used 0.0 to 100.0.
+ *
* @throws IllegalArgumentException if any resource in used has a greater value than the same resource in the total, or used has generic
- * resources that are not present in the total.
+ * resources that are not present in the total.
*/
public double calculateMinPercentageUsedBy(NormalizedResources used, double totalMemoryMb, double usedMemoryMb) {
if (LOG.isTraceEnabled()) {
LOG.trace("Calculating min percentage used by. Used Mem: {} Total Mem: {}"
- + " Used Normalized Resources: {} Total Normalized Resources: {}", totalMemoryMb, usedMemoryMb,
- toNormalizedMap(), used.toNormalizedMap());
+ + " Used Normalized Resources: {} Total Normalized Resources: {}", totalMemoryMb, usedMemoryMb,
+ toNormalizedMap(), used.toNormalizedMap());
}
double min = 1.0;
@@ -322,7 +326,7 @@ public class NormalizedResources {
if (used.otherResources.length > otherResources.length) {
throwBecauseUsedIsNotSubsetOfTotal(used, totalMemoryMb, usedMemoryMb);
}
-
+
for (int i = 0; i < otherResources.length; i++) {
if (otherResources[i] == 0.0) {
//Skip any resources where the total is 0, the percent used for this resource isn't meaningful.
[18/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java b/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java
index f3842b0..9fdd447 100644
--- a/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.blobstore;
import java.io.IOException;
@@ -45,7 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BlobStoreUtils {
- private static final String BLOBSTORE_SUBTREE="/blobstore";
+ private static final String BLOBSTORE_SUBTREE = "/blobstore";
private static final Logger LOG = LoggerFactory.getLogger(BlobStoreUtils.class);
@@ -59,7 +54,8 @@ public class BlobStoreUtils {
Object port = conf.get(Config.STORM_ZOOKEEPER_PORT);
ZookeeperAuthInfo zkAuthInfo = new ZookeeperAuthInfo(conf);
CuratorFramework zkClient = CuratorUtils.newCurator(conf, zkServers, port,
- (String) conf.get(Config.STORM_ZOOKEEPER_ROOT), zkAuthInfo, type.getDefaultZkAcls(conf));
+ (String) conf.get(Config.STORM_ZOOKEEPER_ROOT), zkAuthInfo,
+ type.getDefaultZkAcls(conf));
zkClient.start();
return zkClient;
}
@@ -94,7 +90,7 @@ public class BlobStoreUtils {
int latestSeqNumber = getLatestSequenceNumber(stateInfoList);
LOG.debug("getNimbodesWithLatestSequenceNumberOfBlob stateInfo {} version {}", stateInfoList, latestSeqNumber);
// Get the nimbodes with the latest version
- for(String state : stateInfoList) {
+ for (String state : stateInfoList) {
BlobKeySequenceInfo sequenceInfo = normalizeNimbusHostPortSequenceNumberInfo(state);
if (latestSeqNumber == Integer.parseInt(sequenceInfo.getSequenceNumber())) {
nimbusInfoSet.add(NimbusInfo.parse(sequenceInfo.getNimbusHostPort()));
@@ -122,14 +118,14 @@ public class BlobStoreUtils {
// Download missing blobs from potential nimbodes
public static boolean downloadMissingBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos)
- throws TTransportException {
+ throws TTransportException {
ReadableBlobMeta rbm;
ClientBlobStore remoteBlobStore;
InputStreamWithMeta in;
boolean isSuccess = false;
LOG.debug("Download blob NimbusInfos {}", nimbusInfos);
for (NimbusInfo nimbusInfo : nimbusInfos) {
- if(isSuccess) {
+ if (isSuccess) {
break;
}
LOG.debug("Download blob key: {}, NimbusInfo {}", key, nimbusInfo);
@@ -171,7 +167,7 @@ public class BlobStoreUtils {
// Download updated blobs from potential nimbodes
public static boolean downloadUpdatedBlob(Map<String, Object> conf, BlobStore blobStore, String key, Set<NimbusInfo> nimbusInfos)
- throws TTransportException {
+ throws TTransportException {
ClientBlobStore remoteBlobStore;
InputStreamWithMeta in;
AtomicOutputStream out;
@@ -233,7 +229,8 @@ public class BlobStoreUtils {
cb.createStateInZookeeper(key);
}
- public static void updateKeyForBlobStore (Map<String, Object> conf, BlobStore blobStore, CuratorFramework zkClient, String key, NimbusInfo nimbusDetails) {
+ public static void updateKeyForBlobStore(Map<String, Object> conf, BlobStore blobStore, CuratorFramework zkClient, String key,
+ NimbusInfo nimbusDetails) {
try {
// Most of clojure tests currently try to access the blobs using getBlob. Since, updateKeyForBlobStore
// checks for updating the correct version of the blob as a part of nimbus ha before performing any
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/blobstore/BlobSynchronizer.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/BlobSynchronizer.java b/storm-server/src/main/java/org/apache/storm/blobstore/BlobSynchronizer.java
index 193ccac..19351b1 100644
--- a/storm-server/src/main/java/org/apache/storm/blobstore/BlobSynchronizer.java
+++ b/storm-server/src/main/java/org/apache/storm/blobstore/BlobSynchronizer.java
@@ -1,30 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.blobstore;
import java.nio.channels.ClosedByInterruptException;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-
+import org.apache.curator.framework.CuratorFramework;
import org.apache.storm.generated.KeyNotFoundException;
import org.apache.storm.nimbus.NimbusInfo;
-import org.apache.curator.framework.CuratorFramework;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,14 +44,6 @@ public class BlobSynchronizer {
this.nimbusInfo = nimbusInfo;
}
- public void setZookeeperKeySet(Set<String> zookeeperKeySet) {
- this.zookeeperKeySet = zookeeperKeySet;
- }
-
- public void setBlobStoreKeySet(Set<String> blobStoreKeySet) {
- this.blobStoreKeySet = blobStoreKeySet;
- }
-
public void setZkClient(CuratorFramework zkClient) {
this.zkClient = zkClient;
}
@@ -68,19 +54,28 @@ public class BlobSynchronizer {
return keySet;
}
+ public void setBlobStoreKeySet(Set<String> blobStoreKeySet) {
+ this.blobStoreKeySet = blobStoreKeySet;
+ }
+
public Set<String> getZookeeperKeySet() {
Set<String> keySet = new HashSet<String>();
keySet.addAll(zookeeperKeySet);
return keySet;
}
+ public void setZookeeperKeySet(Set<String> zookeeperKeySet) {
+ this.zookeeperKeySet = zookeeperKeySet;
+ }
+
public synchronized void syncBlobs() {
try {
- LOG.debug("Sync blobs - blobstore keys {}, zookeeper keys {}",getBlobStoreKeySet(), getZookeeperKeySet());
+ LOG.debug("Sync blobs - blobstore keys {}, zookeeper keys {}", getBlobStoreKeySet(), getZookeeperKeySet());
deleteKeySetFromBlobStoreNotOnZookeeper(getBlobStoreKeySet(), getZookeeperKeySet());
updateKeySetForBlobStore(getBlobStoreKeySet());
Set<String> keySetToDownload = getKeySetToDownload(getBlobStoreKeySet(), getZookeeperKeySet());
- LOG.debug("Key set Blobstore-> Zookeeper-> DownloadSet {}-> {}-> {}", getBlobStoreKeySet(), getZookeeperKeySet(), keySetToDownload);
+ LOG.debug("Key set Blobstore-> Zookeeper-> DownloadSet {}-> {}-> {}", getBlobStoreKeySet(), getZookeeperKeySet(),
+ keySetToDownload);
for (String key : keySetToDownload) {
try {
@@ -93,16 +88,16 @@ public class BlobSynchronizer {
LOG.debug("Detected deletion for the key {} while downloading - skipping download", key);
}
}
- } catch(InterruptedException | ClosedByInterruptException exp) {
+ } catch (InterruptedException | ClosedByInterruptException exp) {
LOG.error("Interrupt Exception {}", exp);
- } catch(Exception exp) {
+ } catch (Exception exp) {
throw new RuntimeException(exp);
}
}
public void deleteKeySetFromBlobStoreNotOnZookeeper(Set<String> keySetBlobStore, Set<String> keySetZookeeper) throws Exception {
if (keySetBlobStore.removeAll(keySetZookeeper)
- || (keySetZookeeper.isEmpty() && !keySetBlobStore.isEmpty())) {
+ || (keySetZookeeper.isEmpty() && !keySetBlobStore.isEmpty())) {
LOG.debug("Key set to delete in blobstore {}", keySetBlobStore);
for (String key : keySetBlobStore) {
blobStore.deleteBlob(key, BlobStoreUtils.getNimbusSubject());
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/blobstore/FileBlobStoreImpl.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/FileBlobStoreImpl.java b/storm-server/src/main/java/org/apache/storm/blobstore/FileBlobStoreImpl.java
index 20a389f..782a5bf 100644
--- a/storm-server/src/main/java/org/apache/storm/blobstore/FileBlobStoreImpl.java
+++ b/storm-server/src/main/java/org/apache/storm/blobstore/FileBlobStoreImpl.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.blobstore;
import com.google.common.annotations.VisibleForTesting;
@@ -34,7 +29,6 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Timer;
import java.util.TimerTask;
-
import org.apache.storm.Config;
import org.apache.storm.utils.ObjectReader;
import org.slf4j.Logger;
@@ -48,71 +42,8 @@ public class FileBlobStoreImpl {
private static final int BUCKETS = 1024;
private static final Logger LOG = LoggerFactory.getLogger(FileBlobStoreImpl.class);
private static final Timer timer = new Timer("FileBlobStore cleanup thread", true);
-
- public class KeyInHashDirIterator implements Iterator<String> {
- private int currentBucket = 0;
- private Iterator<String> it = null;
- private String next = null;
-
- public KeyInHashDirIterator() throws IOException {
- primeNext();
- }
-
- private void primeNext() throws IOException {
- while (it == null && currentBucket < BUCKETS) {
- String name = String.valueOf(currentBucket);
- File dir = new File(fullPath, name);
- try {
- it = listKeys(dir);
- } catch (FileNotFoundException e) {
- it = null;
- }
- if (it == null || !it.hasNext()) {
- it = null;
- currentBucket++;
- } else {
- next = it.next();
- }
- }
- }
-
- @Override
- public boolean hasNext() {
- return next != null;
- }
-
- @Override
- public String next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- String current = next;
- next = null;
- if (it != null) {
- if (!it.hasNext()) {
- it = null;
- currentBucket++;
- try {
- primeNext();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- } else {
- next = it.next();
- }
- }
- return current;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("Delete Not Supported");
- }
- }
-
private File fullPath;
private TimerTask cleanup = null;
-
public FileBlobStoreImpl(File path, Map<String, Object> conf) throws IOException {
LOG.info("Creating new blob store based in {}", path);
fullPath = path;
@@ -185,9 +116,9 @@ public class FileBlobStoreImpl {
@VisibleForTesting
File getKeyDir(String key) {
- String hash = String.valueOf(Math.abs((long)key.hashCode()) % BUCKETS);
+ String hash = String.valueOf(Math.abs((long) key.hashCode()) % BUCKETS);
File ret = new File(new File(fullPath, hash), key);
- LOG.debug("{} Looking for {} in {}", new Object[]{fullPath, key, hash});
+ LOG.debug("{} Looking for {} in {}", new Object[]{ fullPath, key, hash });
return ret;
}
@@ -203,7 +134,7 @@ public class FileBlobStoreImpl {
try {
keyDir.delete();
} catch (Exception e) {
- LOG.warn("Could not delete "+keyDir+" will try again later");
+ LOG.warn("Could not delete " + keyDir + " will try again later");
}
}
while (i.hasNext()) {
@@ -221,12 +152,12 @@ public class FileBlobStoreImpl {
ArrayList<LocalFsBlobStoreFile> ret = new ArrayList<LocalFsBlobStoreFile>();
File[] files = path.listFiles();
if (files != null) {
- for (File sub: files) {
+ for (File sub : files) {
try {
ret.add(new LocalFsBlobStoreFile(sub.getParentFile(), sub.getName()));
} catch (IllegalArgumentException e) {
//Ignored the file did not match
- LOG.warn("Found an unexpected file in {} {}",path, sub.getName());
+ LOG.warn("Found an unexpected file in {} {}", path, sub.getName());
}
}
}
@@ -267,4 +198,65 @@ public class FileBlobStoreImpl {
cleanup = null;
}
}
+
+ public class KeyInHashDirIterator implements Iterator<String> {
+ private int currentBucket = 0;
+ private Iterator<String> it = null;
+ private String next = null;
+
+ public KeyInHashDirIterator() throws IOException {
+ primeNext();
+ }
+
+ private void primeNext() throws IOException {
+ while (it == null && currentBucket < BUCKETS) {
+ String name = String.valueOf(currentBucket);
+ File dir = new File(fullPath, name);
+ try {
+ it = listKeys(dir);
+ } catch (FileNotFoundException e) {
+ it = null;
+ }
+ if (it == null || !it.hasNext()) {
+ it = null;
+ currentBucket++;
+ } else {
+ next = it.next();
+ }
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return next != null;
+ }
+
+ @Override
+ public String next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ String current = next;
+ next = null;
+ if (it != null) {
+ if (!it.hasNext()) {
+ it = null;
+ currentBucket++;
+ try {
+ primeNext();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ next = it.next();
+ }
+ }
+ return current;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("Delete Not Supported");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java b/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java
index 570e0ad..31ffae9 100644
--- a/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java
+++ b/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java
@@ -1,28 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.blobstore;
import java.nio.ByteBuffer;
-import java.util.TreeSet;
import java.util.List;
-import java.util.Map;
-
+import java.util.TreeSet;
import org.apache.curator.framework.CuratorFramework;
import org.apache.storm.generated.KeyNotFoundException;
import org.apache.storm.nimbus.NimbusInfo;
@@ -119,7 +111,7 @@ import org.slf4j.LoggerFactory;
*/
public class KeySequenceNumber {
private static final Logger LOG = LoggerFactory.getLogger(KeySequenceNumber.class);
- private final String BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE="/blobstoremaxkeysequencenumber";
+ private final String BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE = "/blobstoremaxkeysequencenumber";
private final String key;
private final NimbusInfo nimbusInfo;
private final int INT_CAPACITY = 4;
@@ -138,7 +130,7 @@ public class KeySequenceNumber {
zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT)
.withACL(ZooDefs.Ids.OPEN_ACL_UNSAFE).forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key);
zkClient.setData().forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key,
- ByteBuffer.allocate(INT_CAPACITY).putInt(INITIAL_SEQUENCE_NUMBER).array());
+ ByteBuffer.allocate(INT_CAPACITY).putInt(INITIAL_SEQUENCE_NUMBER).array());
return INITIAL_SEQUENCE_NUMBER;
}
@@ -157,7 +149,7 @@ public class KeySequenceNumber {
// if not assign the highest sequence number.
for (String stateInfo : stateInfoList) {
sequenceNumbers.add(Integer.parseInt(BlobStoreUtils.normalizeNimbusHostPortSequenceNumberInfo(stateInfo)
- .getSequenceNumber()));
+ .getSequenceNumber()));
}
// Update scenario 2 and 3 explain the code logic written here
@@ -201,7 +193,7 @@ public class KeySequenceNumber {
// there's a race condition with a delete: either blobstore or blobstoremaxsequence
// this should be thrown to the caller to indicate that the key is invalid now
throw new KeyNotFoundException(key);
- } catch(Exception e) {
+ } catch (Exception e) {
// in other case, just set this to 0 to trigger re-sync later
LOG.error("Exception {}", e);
return INITIAL_SEQUENCE_NUMBER - 1;
@@ -210,8 +202,8 @@ public class KeySequenceNumber {
private boolean checkIfStateContainsCurrentNimbusHost(List<String> stateInfoList, NimbusInfo nimbusInfo) {
boolean containsNimbusHost = false;
- for(String stateInfo:stateInfoList) {
- if(stateInfo.contains(nimbusInfo.getHost())) {
+ for (String stateInfo : stateInfoList) {
+ if (stateInfo.contains(nimbusInfo.getHost())) {
containsNimbusHost = true;
break;
}
@@ -221,11 +213,11 @@ public class KeySequenceNumber {
private void incrementMaxSequenceNumber(CuratorFramework zkClient, int count) throws Exception {
zkClient.setData().forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key,
- ByteBuffer.allocate(INT_CAPACITY).putInt(count + 1).array());
+ ByteBuffer.allocate(INT_CAPACITY).putInt(count + 1).array());
}
private int getMaxSequenceNumber(CuratorFramework zkClient) throws Exception {
return ByteBuffer.wrap(zkClient.getData()
- .forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key)).getInt();
+ .forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key)).getInt();
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
index 831fb78..6da1f47 100644
--- a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
+++ b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
@@ -1,55 +1,46 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.blobstore;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.security.auth.Subject;
+import org.apache.curator.framework.CuratorFramework;
import org.apache.storm.cluster.DaemonType;
-import org.apache.storm.generated.SettableBlobMeta;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.KeyAlreadyExistsException;
import org.apache.storm.generated.KeyNotFoundException;
import org.apache.storm.generated.ReadableBlobMeta;
-
+import org.apache.storm.generated.SettableBlobMeta;
import org.apache.storm.nimbus.NimbusInfo;
import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.Utils;
import org.apache.zookeeper.KeeperException.NoNodeException;
-import org.apache.curator.framework.CuratorFramework;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.security.auth.Subject;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import static org.apache.storm.blobstore.BlobStoreAclHandler.ADMIN;
import static org.apache.storm.blobstore.BlobStoreAclHandler.READ;
import static org.apache.storm.blobstore.BlobStoreAclHandler.WRITE;
-import com.google.common.annotations.VisibleForTesting;
-
/**
* Provides a local file system backed blob store implementation for Nimbus.
*
@@ -72,11 +63,11 @@ public class LocalFsBlobStore extends BlobStore {
public static final Logger LOG = LoggerFactory.getLogger(LocalFsBlobStore.class);
private static final String DATA_PREFIX = "data_";
private static final String META_PREFIX = "meta_";
- protected BlobStoreAclHandler _aclHandler;
private final String BLOBSTORE_SUBTREE = "/blobstore/";
+ private final int allPermissions = READ | WRITE | ADMIN;
+ protected BlobStoreAclHandler _aclHandler;
private NimbusInfo nimbusInfo;
private FileBlobStoreImpl fbs;
- private final int allPermissions = READ | WRITE | ADMIN;
private Map<String, Object> conf;
private CuratorFramework zkClient;
@@ -98,22 +89,23 @@ public class LocalFsBlobStore extends BlobStore {
}
@Override
- public AtomicOutputStream createBlob(String key, SettableBlobMeta meta, Subject who) throws AuthorizationException, KeyAlreadyExistsException {
+ public AtomicOutputStream createBlob(String key, SettableBlobMeta meta, Subject who) throws AuthorizationException,
+ KeyAlreadyExistsException {
LOG.debug("Creating Blob for key {}", key);
validateKey(key);
_aclHandler.normalizeSettableBlobMeta(key, meta, who, allPermissions);
BlobStoreAclHandler.validateSettableACLs(key, meta.get_acl());
_aclHandler.hasPermissions(meta.get_acl(), allPermissions, who, key);
- if (fbs.exists(DATA_PREFIX+key)) {
+ if (fbs.exists(DATA_PREFIX + key)) {
throw new KeyAlreadyExistsException(key);
}
BlobStoreFileOutputStream mOut = null;
try {
- mOut = new BlobStoreFileOutputStream(fbs.write(META_PREFIX+key, true));
+ mOut = new BlobStoreFileOutputStream(fbs.write(META_PREFIX + key, true));
mOut.write(Utils.thriftSerialize(meta));
mOut.close();
mOut = null;
- return new BlobStoreFileOutputStream(fbs.write(DATA_PREFIX+key, true));
+ return new BlobStoreFileOutputStream(fbs.write(DATA_PREFIX + key, true));
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
@@ -132,7 +124,7 @@ public class LocalFsBlobStore extends BlobStore {
validateKey(key);
checkPermission(key, who, WRITE);
try {
- return new BlobStoreFileOutputStream(fbs.write(DATA_PREFIX+key, false));
+ return new BlobStoreFileOutputStream(fbs.write(DATA_PREFIX + key, false));
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -141,14 +133,14 @@ public class LocalFsBlobStore extends BlobStore {
private SettableBlobMeta getStoredBlobMeta(String key) throws KeyNotFoundException {
InputStream in = null;
try {
- LocalFsBlobStoreFile pf = fbs.read(META_PREFIX+key);
+ LocalFsBlobStoreFile pf = fbs.read(META_PREFIX + key);
try {
in = pf.getInputStream();
} catch (FileNotFoundException fnf) {
throw new KeyNotFoundException(key);
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
- byte [] buffer = new byte[2048];
+ byte[] buffer = new byte[2048];
int len;
while ((len = in.read(buffer)) > 0) {
out.write(buffer, 0, len);
@@ -172,7 +164,7 @@ public class LocalFsBlobStore extends BlobStore {
@Override
public ReadableBlobMeta getBlobMeta(String key, Subject who) throws AuthorizationException, KeyNotFoundException {
validateKey(key);
- if(!checkForBlobOrDownload(key)) {
+ if (!checkForBlobOrDownload(key)) {
checkForBlobUpdate(key);
}
SettableBlobMeta meta = getStoredBlobMeta(key);
@@ -180,7 +172,7 @@ public class LocalFsBlobStore extends BlobStore {
ReadableBlobMeta rbm = new ReadableBlobMeta();
rbm.set_settable(meta);
try {
- LocalFsBlobStoreFile pf = fbs.read(DATA_PREFIX+key);
+ LocalFsBlobStoreFile pf = fbs.read(DATA_PREFIX + key);
rbm.set_version(pf.getModTime());
} catch (IOException e) {
throw new RuntimeException(e);
@@ -198,7 +190,7 @@ public class LocalFsBlobStore extends BlobStore {
_aclHandler.hasPermissions(orig.get_acl(), ADMIN, who, key);
BlobStoreFileOutputStream mOut = null;
try {
- mOut = new BlobStoreFileOutputStream(fbs.write(META_PREFIX+key, false));
+ mOut = new BlobStoreFileOutputStream(fbs.write(META_PREFIX + key, false));
mOut.write(Utils.thriftSerialize(meta));
mOut.close();
mOut = null;
@@ -233,7 +225,7 @@ public class LocalFsBlobStore extends BlobStore {
// able to delete the blob without checking meta's ACL
// skip checking everything and continue deleting local files
LOG.debug("Given subject is eligible to delete key without checking ACL, skipping... key: {} subject: {}",
- key, who);
+ key, who);
}
try {
@@ -265,13 +257,13 @@ public class LocalFsBlobStore extends BlobStore {
@Override
public InputStreamWithMeta getBlob(String key, Subject who) throws AuthorizationException, KeyNotFoundException {
validateKey(key);
- if(!checkForBlobOrDownload(key)) {
+ if (!checkForBlobOrDownload(key)) {
checkForBlobUpdate(key);
}
SettableBlobMeta meta = getStoredBlobMeta(key);
_aclHandler.hasPermissions(meta.get_acl(), READ, who, key);
try {
- return new BlobStoreFileInputStream(fbs.read(DATA_PREFIX+key));
+ return new BlobStoreFileInputStream(fbs.read(DATA_PREFIX + key));
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -314,7 +306,7 @@ public class LocalFsBlobStore extends BlobStore {
@Override
public int updateBlobReplication(String key, int replication, Subject who) throws AuthorizationException, KeyNotFoundException {
throw new UnsupportedOperationException("For local file system blob store the update blobs function does not work. " +
- "Please use HDFS blob store to make this feature available.");
+ "Please use HDFS blob store to make this feature available.");
}
//This additional check and download is for nimbus high availability in case you have more than one nimbus
@@ -347,7 +339,7 @@ public class LocalFsBlobStore extends BlobStore {
public void fullCleanup(long age) throws IOException {
fbs.fullCleanup(age);
}
-
+
@VisibleForTesting
File getKeyDataDir(String key) {
return fbs.getKeyDir(DATA_PREFIX + key);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreFile.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreFile.java b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreFile.java
index 1b7c969..d160ba6 100644
--- a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreFile.java
+++ b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreFile.java
@@ -1,23 +1,16 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.blobstore;
-import org.apache.storm.generated.SettableBlobMeta;
+package org.apache.storm.blobstore;
import java.io.File;
import java.io.FileInputStream;
@@ -28,14 +21,15 @@ import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.regex.Matcher;
+import org.apache.storm.generated.SettableBlobMeta;
public class LocalFsBlobStoreFile extends BlobStoreFile {
private final String _key;
private final boolean _isTmp;
private final File _path;
- private Long _modTime = null;
private final boolean _mustBeNew;
+ private Long _modTime = null;
private SettableBlobMeta meta;
public LocalFsBlobStoreFile(File base, String name) {
@@ -44,7 +38,7 @@ public class LocalFsBlobStoreFile extends BlobStoreFile {
} else {
Matcher m = TMP_NAME_PATTERN.matcher(name);
if (!m.matches()) {
- throw new IllegalArgumentException("File name does not match '"+name+"' !~ "+TMP_NAME_PATTERN);
+ throw new IllegalArgumentException("File name does not match '" + name + "' !~ " + TMP_NAME_PATTERN);
}
_isTmp = true;
}
@@ -58,7 +52,7 @@ public class LocalFsBlobStoreFile extends BlobStoreFile {
_isTmp = isTmp;
_mustBeNew = mustBeNew;
if (_isTmp) {
- _path = new File(base, System.currentTimeMillis()+TMP_EXT);
+ _path = new File(base, System.currentTimeMillis() + TMP_EXT);
} else {
_path = new File(base, BlobStoreFile.BLOBSTORE_DATA_FILE);
}
@@ -109,7 +103,7 @@ public class LocalFsBlobStoreFile extends BlobStoreFile {
success = _path.createNewFile();
}
if (!success) {
- throw new IOException(_path+" already exists");
+ throw new IOException(_path + " already exists");
}
return new FileOutputStream(_path);
}
@@ -137,18 +131,18 @@ public class LocalFsBlobStoreFile extends BlobStoreFile {
}
@Override
- public SettableBlobMeta getMetadata () {
+ public SettableBlobMeta getMetadata() {
return meta;
}
@Override
- public void setMetadata (SettableBlobMeta meta) {
+ public void setMetadata(SettableBlobMeta meta) {
this.meta = meta;
}
@Override
public String toString() {
- return _path+":"+(_isTmp ? "tmp": BlobStoreFile.BLOBSTORE_DATA_FILE)+":"+_key;
+ return _path + ":" + (_isTmp ? "tmp" : BlobStoreFile.BLOBSTORE_DATA_FILE) + ":" + _key;
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/container/ResourceIsolationInterface.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/container/ResourceIsolationInterface.java b/storm-server/src/main/java/org/apache/storm/container/ResourceIsolationInterface.java
index 733d2ab..8e2ae3c 100644
--- a/storm-server/src/main/java/org/apache/storm/container/ResourceIsolationInterface.java
+++ b/storm-server/src/main/java/org/apache/storm/container/ResourceIsolationInterface.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.container;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/container/cgroup/CgroupManager.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/container/cgroup/CgroupManager.java b/storm-server/src/main/java/org/apache/storm/container/cgroup/CgroupManager.java
index 4079215..a68e18e 100644
--- a/storm-server/src/main/java/org/apache/storm/container/cgroup/CgroupManager.java
+++ b/storm-server/src/main/java/org/apache/storm/container/cgroup/CgroupManager.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.container.cgroup;
@@ -34,7 +28,6 @@ import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
import org.apache.storm.container.ResourceIsolationInterface;
@@ -50,17 +43,40 @@ import org.slf4j.LoggerFactory;
public class CgroupManager implements ResourceIsolationInterface {
private static final Logger LOG = LoggerFactory.getLogger(CgroupManager.class);
-
+ private static final Pattern MEMINFO_PATTERN = Pattern.compile("^([^:\\s]+):\\s*([0-9]+)\\s*kB$");
private CgroupCenter center;
-
private Hierarchy hierarchy;
-
private CgroupCommon rootCgroup;
-
private String rootDir;
-
private Map<String, Object> conf;
+ static long getMemInfoFreeMb() throws IOException {
+ //MemFree: 14367072 kB
+ //Buffers: 536512 kB
+ //Cached: 1192096 kB
+ // MemFree + Buffers + Cached
+ long memFree = 0;
+ long buffers = 0;
+ long cached = 0;
+ try (BufferedReader in = new BufferedReader(new FileReader("/proc/meminfo"))) {
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ Matcher match = MEMINFO_PATTERN.matcher(line);
+ if (match.matches()) {
+ String tag = match.group(1);
+ if (tag.equalsIgnoreCase("MemFree")) {
+ memFree = Long.parseLong(match.group(2));
+ } else if (tag.equalsIgnoreCase("Buffers")) {
+ buffers = Long.parseLong(match.group(2));
+ } else if (tag.equalsIgnoreCase("Cached")) {
+ cached = Long.parseLong(match.group(2));
+ }
+ }
+ }
+ }
+ return (memFree + buffers + cached) / 1024;
+ }
+
/**
* initialize data structures.
*
@@ -102,7 +118,7 @@ public class CgroupManager implements ResourceIsolationInterface {
Set<SubSystemType> types = new HashSet<>();
types.add(SubSystemType.cpu);
this.hierarchy = new Hierarchy(DaemonConfig.getCgroupStormHierarchyName(conf), types,
- DaemonConfig.getCgroupStormHierarchyDir(conf));
+ DaemonConfig.getCgroupStormHierarchyDir(conf));
}
this.rootCgroup =
new CgroupCommon(this.rootDir, this.hierarchy, this.hierarchy.getRootCgroups());
@@ -194,7 +210,7 @@ public class CgroupManager implements ResourceIsolationInterface {
Set<Integer> tasks = workerGroup.getTasks();
if (!tasks.isEmpty()) {
throw new Exception("Cannot correctly shutdown worker CGroup " + workerId + "tasks " + tasks
- + " still running!");
+ + " still running!");
}
this.center.deleteCgroup(workerGroup);
} catch (Exception e) {
@@ -254,35 +270,6 @@ public class CgroupManager implements ResourceIsolationInterface {
return memCore.getPhysicalUsage();
}
- private static final Pattern MEMINFO_PATTERN = Pattern.compile("^([^:\\s]+):\\s*([0-9]+)\\s*kB$");
-
- static long getMemInfoFreeMb() throws IOException {
- //MemFree: 14367072 kB
- //Buffers: 536512 kB
- //Cached: 1192096 kB
- // MemFree + Buffers + Cached
- long memFree = 0;
- long buffers = 0;
- long cached = 0;
- try (BufferedReader in = new BufferedReader(new FileReader("/proc/meminfo"))) {
- String line = null;
- while ((line = in.readLine()) != null) {
- Matcher match = MEMINFO_PATTERN.matcher(line);
- if (match.matches()) {
- String tag = match.group(1);
- if (tag.equalsIgnoreCase("MemFree")) {
- memFree = Long.parseLong(match.group(2));
- } else if (tag.equalsIgnoreCase("Buffers")) {
- buffers = Long.parseLong(match.group(2));
- } else if (tag.equalsIgnoreCase("Cached")) {
- cached = Long.parseLong(match.group(2));
- }
- }
- }
- }
- return (memFree + buffers + cached) / 1024;
- }
-
@Override
public long getSystemFreeMemoryMb() throws IOException {
long rootCgroupLimitFree = Long.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/drpc/BlockingOutstandingRequest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/drpc/BlockingOutstandingRequest.java b/storm-server/src/main/java/org/apache/storm/daemon/drpc/BlockingOutstandingRequest.java
index 489af28..939c1fc 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/drpc/BlockingOutstandingRequest.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/drpc/BlockingOutstandingRequest.java
@@ -15,17 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.daemon.drpc;
import java.util.concurrent.Semaphore;
-
import org.apache.storm.generated.DRPCExceptionType;
import org.apache.storm.generated.DRPCExecutionException;
import org.apache.storm.generated.DRPCRequest;
public class BlockingOutstandingRequest extends OutstandingRequest {
public static final RequestFactory<BlockingOutstandingRequest> FACTORY =
- (function, request) -> new BlockingOutstandingRequest(function, request);
+ (function, request) -> new BlockingOutstandingRequest(function, request);
private Semaphore _sem;
private volatile String _result = null;
private volatile DRPCExecutionException _e = null;
@@ -49,7 +49,7 @@ public class BlockingOutstandingRequest extends OutstandingRequest {
if (_e == null) {
_e = new DRPCExecutionException("Internal Error: No Result and No Exception");
_e.set_type(DRPCExceptionType.INTERNAL_ERROR);
- }
+ }
throw _e;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPC.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPC.java b/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPC.java
index 4093961..94d6b3b 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPC.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPC.java
@@ -15,8 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.daemon.drpc;
+import com.codahale.metrics.Meter;
+import com.google.common.annotations.VisibleForTesting;
import java.security.Principal;
import java.util.HashMap;
import java.util.Map;
@@ -26,7 +29,6 @@ import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.storm.DaemonConfig;
import org.apache.storm.daemon.StormCommon;
import org.apache.storm.generated.AuthorizationException;
@@ -42,26 +44,48 @@ import org.apache.storm.utils.ObjectReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.codahale.metrics.Meter;
-import com.google.common.annotations.VisibleForTesting;
-
public class DRPC implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(DRPC.class);
- private static final DRPCRequest NOTHING_REQUEST = new DRPCRequest("","");
+ private static final DRPCRequest NOTHING_REQUEST = new DRPCRequest("", "");
private static final DRPCExecutionException TIMED_OUT = new DRPCExecutionException("Timed Out");
private static final DRPCExecutionException SHUT_DOWN = new DRPCExecutionException("Server Shutting Down");
private static final DRPCExecutionException DEFAULT_FAILED = new DRPCExecutionException("Request failed");
- static {
- TIMED_OUT.set_type(DRPCExceptionType.SERVER_TIMEOUT);
- SHUT_DOWN.set_type(DRPCExceptionType.SERVER_SHUTDOWN);
- DEFAULT_FAILED.set_type(DRPCExceptionType.FAILED_REQUEST);
- }
private static final Meter meterServerTimedOut = StormMetricsRegistry.registerMeter("drpc:num-server-timedout-requests");
private static final Meter meterExecuteCalls = StormMetricsRegistry.registerMeter("drpc:num-execute-calls");
private static final Meter meterResultCalls = StormMetricsRegistry.registerMeter("drpc:num-result-calls");
private static final Meter meterFailRequestCalls = StormMetricsRegistry.registerMeter("drpc:num-failRequest-calls");
private static final Meter meterFetchRequestCalls = StormMetricsRegistry.registerMeter("drpc:num-fetchRequest-calls");
-
+
+ static {
+ TIMED_OUT.set_type(DRPCExceptionType.SERVER_TIMEOUT);
+ SHUT_DOWN.set_type(DRPCExceptionType.SERVER_SHUTDOWN);
+ DEFAULT_FAILED.set_type(DRPCExceptionType.FAILED_REQUEST);
+ }
+
+ //Waiting to be fetched
+ private final ConcurrentHashMap<String, ConcurrentLinkedQueue<OutstandingRequest>> _queues =
+ new ConcurrentHashMap<>();
+ //Waiting to be returned
+ private final ConcurrentHashMap<String, OutstandingRequest> _requests =
+ new ConcurrentHashMap<>();
+ private final Timer _timer = new Timer();
+ private final AtomicLong _ctr = new AtomicLong(0);
+ private final IAuthorizer _auth;
+
+ public DRPC(Map<String, Object> conf) {
+ this(mkAuthorizationHandler((String) conf.get(DaemonConfig.DRPC_AUTHORIZER), conf),
+ ObjectReader.getInt(conf.get(DaemonConfig.DRPC_REQUEST_TIMEOUT_SECS), 600) * 1000);
+ }
+ public DRPC(IAuthorizer auth, long timeoutMs) {
+ _auth = auth;
+ _timer.scheduleAtFixedRate(new TimerTask() {
+ @Override
+ public void run() {
+ cleanupAll(timeoutMs, TIMED_OUT);
+ }
+ }, timeoutMs / 2, timeoutMs / 2);
+ }
+
private static IAuthorizer mkAuthorizationHandler(String klassname, Map<String, Object> conf) {
try {
return StormCommon.mkAuthorizationHandler(klassname, conf);
@@ -76,7 +100,7 @@ public class DRPC implements AutoCloseable {
private static void logAccess(ReqContext reqContext, String operation, String function) {
ThriftAccessLogger.logAccessFunction(reqContext.requestID(), reqContext.remoteAddress(), reqContext.principal(), operation,
- function);
+ function);
}
@VisibleForTesting
@@ -100,33 +124,7 @@ public class DRPC implements AutoCloseable {
}
}
}
-
- //Waiting to be fetched
- private final ConcurrentHashMap<String, ConcurrentLinkedQueue<OutstandingRequest>> _queues =
- new ConcurrentHashMap<>();
- //Waiting to be returned
- private final ConcurrentHashMap<String, OutstandingRequest> _requests =
- new ConcurrentHashMap<>();
- private final Timer _timer = new Timer();
- private final AtomicLong _ctr = new AtomicLong(0);
- private final IAuthorizer _auth;
-
- public DRPC(Map<String, Object> conf) {
- this(mkAuthorizationHandler((String)conf.get(DaemonConfig.DRPC_AUTHORIZER), conf),
- ObjectReader.getInt(conf.get(DaemonConfig.DRPC_REQUEST_TIMEOUT_SECS), 600) * 1000);
- }
-
- public DRPC(IAuthorizer auth, long timeoutMs) {
- _auth = auth;
- _timer.scheduleAtFixedRate(new TimerTask() {
- @Override
- public void run() {
- cleanupAll(timeoutMs, TIMED_OUT);
- }
- }, timeoutMs/2, timeoutMs/2);
- }
-
-
+
private void checkAuthorization(String operation, String function) throws AuthorizationException {
checkAuthorization(ReqContext.context(), _auth, operation, function);
}
@@ -134,7 +132,7 @@ public class DRPC implements AutoCloseable {
private void checkAuthorizationNoLog(String operation, String function) throws AuthorizationException {
checkAuthorization(ReqContext.context(), _auth, operation, function, false);
}
-
+
private void cleanup(String id) {
OutstandingRequest req = _requests.remove(id);
if (req != null && !req.wasFetched()) {
@@ -152,7 +150,7 @@ public class DRPC implements AutoCloseable {
}
}
}
-
+
private String nextId() {
return String.valueOf(_ctr.incrementAndGet());
}
@@ -207,7 +205,8 @@ public class DRPC implements AutoCloseable {
}
}
- public <T extends OutstandingRequest> T execute(String functionName, String funcArgs, RequestFactory<T> factory) throws AuthorizationException {
+ public <T extends OutstandingRequest> T execute(String functionName, String funcArgs, RequestFactory<T> factory) throws
+ AuthorizationException {
meterExecuteCalls.mark();
checkAuthorization("execute", functionName);
String id = nextId();
@@ -218,11 +217,11 @@ public class DRPC implements AutoCloseable {
q.add(req);
return req;
}
-
+
public String executeBlocking(String functionName, String funcArgs) throws DRPCExecutionException, AuthorizationException {
BlockingOutstandingRequest req = execute(functionName, funcArgs, BlockingOutstandingRequest.FACTORY);
try {
- LOG.debug("Waiting for result {} {}",functionName, funcArgs);
+ LOG.debug("Waiting for result {} {}", functionName, funcArgs);
return req.getResult();
} catch (DRPCExecutionException e) {
throw e;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPCThrift.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPCThrift.java b/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPCThrift.java
index 2d88e8e..01d2392 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPCThrift.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/drpc/DRPCThrift.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.daemon.drpc;
import org.apache.storm.generated.AuthorizationException;
@@ -52,7 +53,7 @@ public class DRPCThrift implements DistributedRPC.Iface, DistributedRPCInvocatio
@Override
public String execute(String functionName, String funcArgs)
- throws DRPCExecutionException, AuthorizationException {
+ throws DRPCExecutionException, AuthorizationException {
return _drpc.executeBlocking(functionName, funcArgs);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/drpc/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/drpc/OutstandingRequest.java b/storm-server/src/main/java/org/apache/storm/daemon/drpc/OutstandingRequest.java
index adb916f..06c596e 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/drpc/OutstandingRequest.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/drpc/OutstandingRequest.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.daemon.drpc;
import org.apache.storm.generated.DRPCExecutionException;
@@ -42,7 +43,7 @@ public abstract class OutstandingRequest {
}
public boolean wasFetched() {
- return _fetched;
+ return _fetched;
}
public String getFunction() {
@@ -54,5 +55,6 @@ public abstract class OutstandingRequest {
}
public abstract void returnResult(String result);
+
public abstract void fail(DRPCExecutionException e);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/drpc/RequestFactory.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/drpc/RequestFactory.java b/storm-server/src/main/java/org/apache/storm/daemon/drpc/RequestFactory.java
index 6a0e610..e6cd799 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/drpc/RequestFactory.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/drpc/RequestFactory.java
@@ -15,10 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.daemon.drpc;
import org.apache.storm.generated.DRPCRequest;
public interface RequestFactory<T extends OutstandingRequest> {
- public T mkRequest(String function, DRPCRequest req);
+ public T mkRequest(String function, DRPCRequest req);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/metrics/MetricsUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/metrics/MetricsUtils.java b/storm-server/src/main/java/org/apache/storm/daemon/metrics/MetricsUtils.java
index 85ee27f..abd7cb1 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/metrics/MetricsUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/metrics/MetricsUtils.java
@@ -22,7 +22,6 @@ import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.DaemonConfig;
import org.apache.storm.daemon.metrics.reporters.JmxPreparableReporter;
import org.apache.storm.daemon.metrics.reporters.PreparableReporter;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/ConsolePreparableReporter.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/ConsolePreparableReporter.java b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/ConsolePreparableReporter.java
index cd6d069..47e4f24 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/ConsolePreparableReporter.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/ConsolePreparableReporter.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.daemon.metrics.reporters;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/CsvPreparableReporter.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/CsvPreparableReporter.java b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/CsvPreparableReporter.java
index 3ab6a99..4952051 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/CsvPreparableReporter.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/CsvPreparableReporter.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.daemon.metrics.reporters;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java
index c369879..3fc77d1 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/JmxPreparableReporter.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.daemon.metrics.reporters;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/PreparableReporter.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/PreparableReporter.java b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/PreparableReporter.java
index 958a01d..0e1a6e3 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/PreparableReporter.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/metrics/reporters/PreparableReporter.java
@@ -1,32 +1,28 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.metrics.reporters;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Reporter;
-
import java.io.Closeable;
import java.util.Map;
public interface PreparableReporter<T extends Reporter & Closeable> {
- void prepare(MetricRegistry metricsRegistry, Map<String, Object> topoConf);
- void start();
- void stop();
+ void prepare(MetricRegistry metricsRegistry, Map<String, Object> topoConf);
+
+ void start();
+
+ void stop();
}
[24/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-eventhubs
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventHubReceiverFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventHubReceiverFactory.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventHubReceiverFactory.java
index a4901ae..d25ebdd 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventHubReceiverFactory.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IEventHubReceiverFactory.java
@@ -1,23 +1,18 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *******************************************************************************/
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
/**
- *
+ *
*/
+
package org.apache.storm.eventhubs.spout;
import java.io.Serializable;
@@ -26,5 +21,5 @@ import java.io.Serializable;
* An abstract factory to generate EventHubReceiver
*/
public interface IEventHubReceiverFactory extends Serializable {
- IEventHubReceiver create(EventHubSpoutConfig config, String partitionId);
+ IEventHubReceiver create(EventHubSpoutConfig config, String partitionId);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionCoordinator.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionCoordinator.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionCoordinator.java
index e99f20a..d83c0cb 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionCoordinator.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionCoordinator.java
@@ -15,13 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import java.util.List;
public interface IPartitionCoordinator {
- List<IPartitionManager> getMyPartitionManagers();
+ List<IPartitionManager> getMyPartitionManagers();
- IPartitionManager getPartitionManager(String partitionId);
+ IPartitionManager getPartitionManager(String partitionId);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionManager.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionManager.java
index d391edd..345fd48 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionManager.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionManager.java
@@ -15,23 +15,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import java.util.Map;
public interface IPartitionManager {
- void open() throws Exception;
+ void open() throws Exception;
+
+ void close();
- void close();
+ EventDataWrap receive();
- EventDataWrap receive();
+ void checkpoint();
- void checkpoint();
+ void ack(String offset);
- void ack(String offset);
+ void fail(String offset);
- void fail(String offset);
-
- Map<String, Object> getMetricsData();
+ Map<String, Object> getMetricsData();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionManagerFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionManagerFactory.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionManagerFactory.java
index dc136eb..c3ce2ec 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionManagerFactory.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IPartitionManagerFactory.java
@@ -1,23 +1,18 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *******************************************************************************/
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
/**
- *
+ *
*/
+
package org.apache.storm.eventhubs.spout;
import java.io.Serializable;
@@ -26,8 +21,8 @@ import java.io.Serializable;
* An interface of factory method to create IPartitionManager
*/
public interface IPartitionManagerFactory extends Serializable {
- IPartitionManager create(EventHubSpoutConfig spoutConfig,
- String partitionId,
- IStateStore stateStore,
- IEventHubReceiver receiver);
+ IPartitionManager create(EventHubSpoutConfig spoutConfig,
+ String partitionId,
+ IStateStore stateStore,
+ IEventHubReceiver receiver);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IStateStore.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IStateStore.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IStateStore.java
index 03c7ae8..311ceb7 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IStateStore.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/IStateStore.java
@@ -15,17 +15,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import java.io.Serializable;
public interface IStateStore extends Serializable {
- public void open();
+ public void open();
- public void close();
+ public void close();
- public void saveData(String path, String data);
+ public void saveData(String path, String data);
- public String readData(String path);
+ public String readData(String path);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/MessageId.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/MessageId.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/MessageId.java
index 59d5c71..0bb1f10 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/MessageId.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/MessageId.java
@@ -15,42 +15,43 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
public class MessageId {
- private final String partitionId;
- private final String offset;
- private final long sequenceNumber;
-
- public MessageId(
- String partitionId,
- String offset,
- long sequenceNumber) {
- this.partitionId = partitionId;
- this.offset = offset;
- this.sequenceNumber = sequenceNumber;
- }
-
- public static MessageId create(String partitionId, String offset, long sequenceNumber) {
- return new MessageId(partitionId, offset, sequenceNumber);
- }
-
- public String getPartitionId() {
- return this.partitionId;
- }
-
- public String getOffset() {
- return this.offset;
- }
-
- public Long getSequenceNumber() {
- return this.sequenceNumber;
- }
-
- @Override
- public String toString() {
- return String.format("PartitionId: %s, Offset: %s, SequenceNumber: %s",
- this.partitionId, this.offset, this.sequenceNumber);
- }
+ private final String partitionId;
+ private final String offset;
+ private final long sequenceNumber;
+
+ public MessageId(
+ String partitionId,
+ String offset,
+ long sequenceNumber) {
+ this.partitionId = partitionId;
+ this.offset = offset;
+ this.sequenceNumber = sequenceNumber;
+ }
+
+ public static MessageId create(String partitionId, String offset, long sequenceNumber) {
+ return new MessageId(partitionId, offset, sequenceNumber);
+ }
+
+ public String getPartitionId() {
+ return this.partitionId;
+ }
+
+ public String getOffset() {
+ return this.offset;
+ }
+
+ public Long getSequenceNumber() {
+ return this.sequenceNumber;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("PartitionId: %s, Offset: %s, SequenceNumber: %s",
+ this.partitionId, this.offset, this.sequenceNumber);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/PartitionManager.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/PartitionManager.java
index 20e021a..c5c5084 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/PartitionManager.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/PartitionManager.java
@@ -15,87 +15,87 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
-package org.apache.storm.eventhubs.spout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+package org.apache.storm.eventhubs.spout;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.TreeSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class PartitionManager extends SimplePartitionManager {
- private static final Logger logger = LoggerFactory.getLogger(PartitionManager.class);
- private final int ehReceiveTimeoutMs = 5000;
+ private static final Logger logger = LoggerFactory.getLogger(PartitionManager.class);
+ private final int ehReceiveTimeoutMs = 5000;
- //all sent events are stored in pending
- private final Map<String, EventDataWrap> pending;
- //all failed events are put in toResend, which is sorted by event's offset
- private final TreeSet<EventDataWrap> toResend;
+ //all sent events are stored in pending
+ private final Map<String, EventDataWrap> pending;
+ //all failed events are put in toResend, which is sorted by event's offset
+ private final TreeSet<EventDataWrap> toResend;
- public PartitionManager(
- EventHubSpoutConfig spoutConfig,
- String partitionId,
- IStateStore stateStore,
- IEventHubReceiver receiver) {
+ public PartitionManager(
+ EventHubSpoutConfig spoutConfig,
+ String partitionId,
+ IStateStore stateStore,
+ IEventHubReceiver receiver) {
- super(spoutConfig, partitionId, stateStore, receiver);
-
- this.pending = new LinkedHashMap<String, EventDataWrap>();
- this.toResend = new TreeSet<EventDataWrap>();
- }
+ super(spoutConfig, partitionId, stateStore, receiver);
- @Override
- public EventDataWrap receive() {
- if(pending.size() >= config.getMaxPendingMsgsPerPartition()) {
- return null;
+ this.pending = new LinkedHashMap<String, EventDataWrap>();
+ this.toResend = new TreeSet<EventDataWrap>();
}
- EventDataWrap eventDatawrap;
- if (toResend.isEmpty()) {
- eventDatawrap = receiver.receive();
- } else {
- eventDatawrap = toResend.pollFirst();
- }
+ @Override
+ public EventDataWrap receive() {
+ if (pending.size() >= config.getMaxPendingMsgsPerPartition()) {
+ return null;
+ }
- if (eventDatawrap != null) {
- lastOffset = eventDatawrap.getMessageId().getOffset();
- pending.put(lastOffset, eventDatawrap);
- }
+ EventDataWrap eventDatawrap;
+ if (toResend.isEmpty()) {
+ eventDatawrap = receiver.receive();
+ } else {
+ eventDatawrap = toResend.pollFirst();
+ }
- return eventDatawrap;
- }
+ if (eventDatawrap != null) {
+ lastOffset = eventDatawrap.getMessageId().getOffset();
+ pending.put(lastOffset, eventDatawrap);
+ }
- @Override
- public void ack(String offset) {
- pending.remove(offset);
- }
+ return eventDatawrap;
+ }
- @Override
- public void fail(String offset) {
- logger.warn("fail on " + offset);
- EventDataWrap eventDataWrap = pending.remove(offset);
- toResend.add(eventDataWrap);
- }
-
- @Override
- protected String getCompletedOffset() {
- String offset = null;
-
- if(pending.size() > 0) {
- //find the smallest offset in pending list
- offset = pending.keySet().iterator().next();
+ @Override
+ public void ack(String offset) {
+ pending.remove(offset);
}
- if(toResend.size() > 0) {
- //find the smallest offset in toResend list
- String offset2 = toResend.first().getMessageId().getOffset();
- if(offset == null || offset2.compareTo(offset) < 0) {
- offset = offset2;
- }
+
+ @Override
+ public void fail(String offset) {
+ logger.warn("fail on " + offset);
+ EventDataWrap eventDataWrap = pending.remove(offset);
+ toResend.add(eventDataWrap);
}
- if(offset == null) {
- offset = lastOffset;
+
+ @Override
+ protected String getCompletedOffset() {
+ String offset = null;
+
+ if (pending.size() > 0) {
+ //find the smallest offset in pending list
+ offset = pending.keySet().iterator().next();
+ }
+ if (toResend.size() > 0) {
+ //find the smallest offset in toResend list
+ String offset2 = toResend.first().getMessageId().getOffset();
+ if (offset == null || offset2.compareTo(offset) < 0) {
+ offset = offset2;
+ }
+ }
+ if (offset == null) {
+ offset = lastOffset;
+ }
+ return offset;
}
- return offset;
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/SimplePartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/SimplePartitionManager.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/SimplePartitionManager.java
index 25c4261..436aabe 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/SimplePartitionManager.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/SimplePartitionManager.java
@@ -15,119 +15,119 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
+import java.time.Instant;
+import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.time.Instant;
-import java.util.Map;
/**
* A simple partition manager that does not re-send failed messages
*/
public class SimplePartitionManager implements IPartitionManager {
- private static final Logger logger = LoggerFactory.getLogger(SimplePartitionManager.class);
- private static final String statePathPrefix = "/eventhubspout";
-
- protected final IEventHubReceiver receiver;
- protected String lastOffset = "-1";
- protected String committedOffset = "-1";
-
- protected final EventHubSpoutConfig config;
- private final String partitionId;
- private final IStateStore stateStore;
- private final String statePath;
-
- public SimplePartitionManager(
- EventHubSpoutConfig spoutConfig,
- String partitionId,
- IStateStore stateStore,
- IEventHubReceiver receiver) {
- this.receiver = receiver;
- this.config = spoutConfig;
- this.partitionId = partitionId;
- this.statePath = this.getPartitionStatePath();
- this.stateStore = stateStore;
- }
-
- @Override
- public void open() throws Exception {
-
- //read from state store, if not found, use startingOffset
- String offset = stateStore.readData(statePath);
- logger.info("read offset from state store: " + offset);
- if(offset == null) {
- offset = FieldConstants.DefaultStartingOffset;
+ private static final Logger logger = LoggerFactory.getLogger(SimplePartitionManager.class);
+ private static final String statePathPrefix = "/eventhubspout";
+
+ protected final IEventHubReceiver receiver;
+ protected final EventHubSpoutConfig config;
+ private final String partitionId;
+ private final IStateStore stateStore;
+ private final String statePath;
+ protected String lastOffset = "-1";
+ protected String committedOffset = "-1";
+
+ public SimplePartitionManager(
+ EventHubSpoutConfig spoutConfig,
+ String partitionId,
+ IStateStore stateStore,
+ IEventHubReceiver receiver) {
+ this.receiver = receiver;
+ this.config = spoutConfig;
+ this.partitionId = partitionId;
+ this.statePath = this.getPartitionStatePath();
+ this.stateStore = stateStore;
+ }
+
+ @Override
+ public void open() throws Exception {
+
+ //read from state store, if not found, use startingOffset
+ String offset = stateStore.readData(statePath);
+ logger.info("read offset from state store: " + offset);
+ if (offset == null) {
+ offset = FieldConstants.DefaultStartingOffset;
+ }
+ IEventFilter filter;
+ if (offset.equals(FieldConstants.DefaultStartingOffset)
+ && config.getEnqueueTimeFilter() != 0) {
+ filter = new EventHubFilter(Instant.ofEpochMilli(config.getEnqueueTimeFilter()));
+ } else {
+ filter = new EventHubFilter(offset);
+ }
+
+ receiver.open(filter);
+ }
+
+ @Override
+ public void close() {
+ this.receiver.close();
+ this.checkpoint();
}
- IEventFilter filter;
- if (offset.equals(FieldConstants.DefaultStartingOffset)
- && config.getEnqueueTimeFilter() != 0) {
- filter = new EventHubFilter(Instant.ofEpochMilli(config.getEnqueueTimeFilter()));
+
+ @Override
+ public void checkpoint() {
+ String completedOffset = getCompletedOffset();
+ if (!committedOffset.equals(completedOffset)) {
+ logger.info("saving state " + completedOffset);
+ stateStore.saveData(statePath, completedOffset);
+ committedOffset = completedOffset;
+ }
+ }
+
+ protected String getCompletedOffset() {
+ return lastOffset;
}
- else{
- filter = new EventHubFilter(offset);
+
+ @Override
+ public EventDataWrap receive() {
+ EventDataWrap eventDatawrap = receiver.receive();
+ if (eventDatawrap != null) {
+ lastOffset = eventDatawrap.getEventData().getSystemProperties().getOffset();
+ }
+ return eventDatawrap;
}
- receiver.open(filter);
- }
-
- @Override
- public void close() {
- this.receiver.close();
- this.checkpoint();
- }
-
- @Override
- public void checkpoint() {
- String completedOffset = getCompletedOffset();
- if(!committedOffset.equals(completedOffset)) {
- logger.info("saving state " + completedOffset);
- stateStore.saveData(statePath, completedOffset);
- committedOffset = completedOffset;
+ @Override
+ public void ack(String offset) {
+ //do nothing
}
- }
-
- protected String getCompletedOffset() {
- return lastOffset;
- }
-
- @Override
- public EventDataWrap receive() {
- EventDataWrap eventDatawrap = receiver.receive();
- if (eventDatawrap != null) {
- lastOffset = eventDatawrap.getEventData().getSystemProperties().getOffset();
+
+ @Override
+ public void fail(String offset) {
+ logger.warn("fail on " + offset);
+ //do nothing
+ }
+
+ private String getPartitionStatePath() {
+
+ // Partition state path =
+ // "/{prefix}/{topologyName}/{namespace}/{entityPath}/partitions/{partitionId}/state";
+ String namespace = config.getNamespace();
+ String entityPath = config.getEntityPath();
+ String topologyName = config.getTopologyName();
+
+ String partitionStatePath =
+ statePathPrefix + "/" + topologyName + "/" + namespace + "/" + entityPath + "/partitions/" + this.partitionId;
+
+ logger.info("partition state path: " + partitionStatePath);
+
+ return partitionStatePath;
+ }
+
+ @Override
+ public Map<String, Object> getMetricsData() {
+ return receiver.getMetricsData();
}
- return eventDatawrap;
- }
-
- @Override
- public void ack(String offset) {
- //do nothing
- }
-
- @Override
- public void fail(String offset) {
- logger.warn("fail on " + offset);
- //do nothing
- }
-
- private String getPartitionStatePath() {
-
- // Partition state path =
- // "/{prefix}/{topologyName}/{namespace}/{entityPath}/partitions/{partitionId}/state";
- String namespace = config.getNamespace();
- String entityPath = config.getEntityPath();
- String topologyName = config.getTopologyName();
-
- String partitionStatePath = statePathPrefix + "/" + topologyName + "/" + namespace + "/" + entityPath + "/partitions/" + this.partitionId;
-
- logger.info("partition state path: " + partitionStatePath);
-
- return partitionStatePath;
- }
-
- @Override
- public Map<String, Object> getMetricsData() {
- return receiver.getMetricsData();
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/StaticPartitionCoordinator.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/StaticPartitionCoordinator.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/StaticPartitionCoordinator.java
index abbecac..54c0f7d 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/StaticPartitionCoordinator.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/StaticPartitionCoordinator.java
@@ -15,70 +15,70 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class StaticPartitionCoordinator implements IPartitionCoordinator {
- private static final Logger logger = LoggerFactory.getLogger(StaticPartitionCoordinator.class);
+ private static final Logger logger = LoggerFactory.getLogger(StaticPartitionCoordinator.class);
- protected final EventHubSpoutConfig config;
- protected final int taskIndex;
- protected final int totalTasks;
- protected final List<IPartitionManager> partitionManagers;
- protected final Map<String, IPartitionManager> partitionManagerMap;
- protected final IStateStore stateStore;
+ protected final EventHubSpoutConfig config;
+ protected final int taskIndex;
+ protected final int totalTasks;
+ protected final List<IPartitionManager> partitionManagers;
+ protected final Map<String, IPartitionManager> partitionManagerMap;
+ protected final IStateStore stateStore;
- public StaticPartitionCoordinator(
- EventHubSpoutConfig spoutConfig,
- int taskIndex,
- int totalTasks,
- IStateStore stateStore,
- IPartitionManagerFactory pmFactory,
- IEventHubReceiverFactory recvFactory) {
+ public StaticPartitionCoordinator(
+ EventHubSpoutConfig spoutConfig,
+ int taskIndex,
+ int totalTasks,
+ IStateStore stateStore,
+ IPartitionManagerFactory pmFactory,
+ IEventHubReceiverFactory recvFactory) {
- this.config = spoutConfig;
- this.taskIndex = taskIndex;
- this.totalTasks = totalTasks;
- this.stateStore = stateStore;
- List<String> partitionIds = calculateParititionIdsToOwn();
- partitionManagerMap = new HashMap<String, IPartitionManager>();
- partitionManagers = new ArrayList<IPartitionManager>();
-
- for (String partitionId : partitionIds) {
- IEventHubReceiver receiver = recvFactory.create(config, partitionId);
- IPartitionManager partitionManager = pmFactory.create(
- config, partitionId, stateStore, receiver);
- partitionManagerMap.put(partitionId, partitionManager);
- partitionManagers.add(partitionManager);
- }
- }
+ this.config = spoutConfig;
+ this.taskIndex = taskIndex;
+ this.totalTasks = totalTasks;
+ this.stateStore = stateStore;
+ List<String> partitionIds = calculateParititionIdsToOwn();
+ partitionManagerMap = new HashMap<String, IPartitionManager>();
+ partitionManagers = new ArrayList<IPartitionManager>();
- @Override
- public List<IPartitionManager> getMyPartitionManagers() {
- return partitionManagers;
- }
+ for (String partitionId : partitionIds) {
+ IEventHubReceiver receiver = recvFactory.create(config, partitionId);
+ IPartitionManager partitionManager = pmFactory.create(
+ config, partitionId, stateStore, receiver);
+ partitionManagerMap.put(partitionId, partitionManager);
+ partitionManagers.add(partitionManager);
+ }
+ }
- @Override
- public IPartitionManager getPartitionManager(String partitionId) {
- return partitionManagerMap.get(partitionId);
- }
+ @Override
+ public List<IPartitionManager> getMyPartitionManagers() {
+ return partitionManagers;
+ }
- protected List<String> calculateParititionIdsToOwn() {
- List<String> taskPartitions = new ArrayList<String>();
- for (int i = this.taskIndex; i < config.getPartitionCount(); i += this.totalTasks) {
- taskPartitions.add(Integer.toString(i));
- logger.info(String.format("taskIndex %d owns partitionId %d.", this.taskIndex, i));
+ @Override
+ public IPartitionManager getPartitionManager(String partitionId) {
+ return partitionManagerMap.get(partitionId);
}
- return taskPartitions;
- }
+ protected List<String> calculateParititionIdsToOwn() {
+ List<String> taskPartitions = new ArrayList<String>();
+ for (int i = this.taskIndex; i < config.getPartitionCount(); i += this.totalTasks) {
+ taskPartitions.add(Integer.toString(i));
+ logger.info(String.format("taskIndex %d owns partitionId %d.", this.taskIndex, i));
+ }
+
+ return taskPartitions;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/StringEventDataScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/StringEventDataScheme.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/StringEventDataScheme.java
index a2024bd..5b6a2b9 100644
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/StringEventDataScheme.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/StringEventDataScheme.java
@@ -15,60 +15,58 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import com.microsoft.azure.eventhubs.EventData;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.storm.tuple.Fields;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.List;
-
/**
- * An Event Data Scheme which deserializes message payload into the Strings.
- * No encoding is assumed. The receiver will need to handle parsing of the
- * string data in appropriate encoding.
+ * An Event Data Scheme which deserializes message payload into the Strings. No encoding is assumed. The receiver will need to handle
+ * parsing of the string data in appropriate encoding.
+ *
+ * Note: Unlike other schemes provided, this scheme does not include any metadata.
*
- * Note: Unlike other schemes provided, this scheme does not include any
- * metadata.
- *
- * For metadata please refer to {@link BinaryEventDataScheme}, {@link EventDataScheme}
+ * For metadata please refer to {@link BinaryEventDataScheme}, {@link EventDataScheme}
*/
public class StringEventDataScheme implements IEventDataScheme {
- private static final long serialVersionUID = 1L;
- private static final Logger logger = LoggerFactory.getLogger(StringEventDataScheme.class);
+ private static final long serialVersionUID = 1L;
+ private static final Logger logger = LoggerFactory.getLogger(StringEventDataScheme.class);
- @Override
- public List<Object> deserialize(EventData eventData) {
- final List<Object> fieldContents = new ArrayList<Object>();
- String messageData = "";
- if (eventData.getBytes()!=null) {
- messageData = new String(eventData.getBytes());
- }
- /*Will only serialize AMQPValue type*/
- else if (eventData.getObject()!=null) {
- try {
- if (!(eventData.getObject() instanceof List)) {
- messageData = eventData.getObject().toString();
- } else {
- throw new RuntimeException("Cannot serialize the given AMQP type.");
+ @Override
+ public List<Object> deserialize(EventData eventData) {
+ final List<Object> fieldContents = new ArrayList<Object>();
+ String messageData = "";
+ if (eventData.getBytes() != null) {
+ messageData = new String(eventData.getBytes());
}
- } catch (RuntimeException e){
- logger.error("Failed to serialize EventData payload class"
- + eventData.getObject().getClass());
- logger.error("Exception encountered while serializing EventData payload is"
- + e.toString());
- throw e;
- }
+ /*Will only serialize AMQPValue type*/
+ else if (eventData.getObject() != null) {
+ try {
+ if (!(eventData.getObject() instanceof List)) {
+ messageData = eventData.getObject().toString();
+ } else {
+ throw new RuntimeException("Cannot serialize the given AMQP type.");
+ }
+ } catch (RuntimeException e) {
+ logger.error("Failed to serialize EventData payload class"
+ + eventData.getObject().getClass());
+ logger.error("Exception encountered while serializing EventData payload is"
+ + e.toString());
+ throw e;
+ }
+ }
+ fieldContents.add(messageData);
+ return fieldContents;
}
- fieldContents.add(messageData);
- return fieldContents;
- }
- @Override
- public Fields getOutputFields() {
- return new Fields(FieldConstants.Message);
- }
+ @Override
+ public Fields getOutputFields() {
+ return new Fields(FieldConstants.Message);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/ZookeeperStateStore.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/ZookeeperStateStore.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/ZookeeperStateStore.java
index 063aa4d..4e2ece9 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/ZookeeperStateStore.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/spout/ZookeeperStateStore.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import org.apache.curator.RetryPolicy;
@@ -25,71 +26,71 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ZookeeperStateStore implements IStateStore {
- private static final long serialVersionUID = 1L;
- private static final Logger logger = LoggerFactory.getLogger(ZookeeperStateStore.class);
+ private static final long serialVersionUID = 1L;
+ private static final Logger logger = LoggerFactory.getLogger(ZookeeperStateStore.class);
- private final String zookeeperConnectionString;
- private final CuratorFramework curatorFramework;
-
- public ZookeeperStateStore(String zookeeperConnectionString) {
- this(zookeeperConnectionString, 3, 100);
- }
+ private final String zookeeperConnectionString;
+ private final CuratorFramework curatorFramework;
- public ZookeeperStateStore(String connectionString, int retries, int retryInterval) {
- if (connectionString == null) {
- zookeeperConnectionString = "localhost:2181";
- } else {
- zookeeperConnectionString = connectionString;
+ public ZookeeperStateStore(String zookeeperConnectionString) {
+ this(zookeeperConnectionString, 3, 100);
}
- RetryPolicy retryPolicy = new RetryNTimes(retries, retryInterval);
- curatorFramework = CuratorFrameworkFactory.newClient(zookeeperConnectionString, retryPolicy);
- }
+ public ZookeeperStateStore(String connectionString, int retries, int retryInterval) {
+ if (connectionString == null) {
+ zookeeperConnectionString = "localhost:2181";
+ } else {
+ zookeeperConnectionString = connectionString;
+ }
- @Override
- public void open() {
- curatorFramework.start();
- }
+ RetryPolicy retryPolicy = new RetryNTimes(retries, retryInterval);
+ curatorFramework = CuratorFrameworkFactory.newClient(zookeeperConnectionString, retryPolicy);
+ }
- @Override
- public void close() {
- curatorFramework.close();
- }
+ @Override
+ public void open() {
+ curatorFramework.start();
+ }
+
+ @Override
+ public void close() {
+ curatorFramework.close();
+ }
- @Override
- public void saveData(String statePath, String data) {
- data = data == null ? "" : data;
- byte[] bytes = data.getBytes();
+ @Override
+ public void saveData(String statePath, String data) {
+ data = data == null ? "" : data;
+ byte[] bytes = data.getBytes();
- try {
- if (curatorFramework.checkExists().forPath(statePath) == null) {
- curatorFramework.create().creatingParentsIfNeeded().forPath(statePath, bytes);
- } else {
- curatorFramework.setData().forPath(statePath, bytes);
- }
+ try {
+ if (curatorFramework.checkExists().forPath(statePath) == null) {
+ curatorFramework.create().creatingParentsIfNeeded().forPath(statePath, bytes);
+ } else {
+ curatorFramework.setData().forPath(statePath, bytes);
+ }
- logger.info(String.format("data was saved. path: %s, data: %s.", statePath, data));
- } catch (Exception e) {
- throw new RuntimeException(e);
+ logger.info(String.format("data was saved. path: %s, data: %s.", statePath, data));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
- }
- @Override
- public String readData(String statePath) {
- try {
- if (curatorFramework.checkExists().forPath(statePath) == null) {
- // do we want to throw an exception if path doesn't exist??
- return null;
- } else {
- byte[] bytes = curatorFramework.getData().forPath(statePath);
- String data = new String(bytes);
+ @Override
+ public String readData(String statePath) {
+ try {
+ if (curatorFramework.checkExists().forPath(statePath) == null) {
+ // do we want to throw an exception if path doesn't exist??
+ return null;
+ } else {
+ byte[] bytes = curatorFramework.getData().forPath(statePath);
+ String data = new String(bytes);
- logger.info(String.format("data was retrieved. path: %s, data: %s.", statePath, data));
+ logger.info(String.format("data was retrieved. path: %s, data: %s.", statePath, data));
- return data;
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
+ return data;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Coordinator.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Coordinator.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Coordinator.java
index 253b317..b87c81b 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Coordinator.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Coordinator.java
@@ -15,46 +15,45 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.trident;
+import org.apache.storm.eventhubs.spout.EventHubSpoutConfig;
+import org.apache.storm.trident.spout.IOpaquePartitionedTridentSpout;
+import org.apache.storm.trident.spout.IPartitionedTridentSpout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.storm.trident.spout.IOpaquePartitionedTridentSpout;
-import org.apache.storm.trident.spout.IPartitionedTridentSpout;
+public class Coordinator implements IPartitionedTridentSpout.Coordinator<Partitions>,
+ IOpaquePartitionedTridentSpout.Coordinator<Partitions> {
+ private static final Logger logger = LoggerFactory.getLogger(Coordinator.class);
+ private final EventHubSpoutConfig spoutConfig;
+ Partitions partitions;
-import org.apache.storm.eventhubs.spout.EventHubSpoutConfig;
+ public Coordinator(EventHubSpoutConfig spoutConfig) {
+ this.spoutConfig = spoutConfig;
+ }
-public class Coordinator implements IPartitionedTridentSpout.Coordinator<Partitions>,
- IOpaquePartitionedTridentSpout.Coordinator<Partitions> {
- private static final Logger logger = LoggerFactory.getLogger(Coordinator.class);
- private final EventHubSpoutConfig spoutConfig;
- Partitions partitions;
-
- public Coordinator(EventHubSpoutConfig spoutConfig) {
- this.spoutConfig = spoutConfig;
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public Partitions getPartitionsForBatch() {
- if(partitions != null) {
- return partitions;
+ @Override
+ public void close() {
}
-
- partitions = new Partitions();
- for(int i=0; i<spoutConfig.getPartitionCount(); ++i) {
- partitions.addPartition(new Partition(spoutConfig, Integer.toString(i)));
+
+ @Override
+ public Partitions getPartitionsForBatch() {
+ if (partitions != null) {
+ return partitions;
+ }
+
+ partitions = new Partitions();
+ for (int i = 0; i < spoutConfig.getPartitionCount(); ++i) {
+ partitions.addPartition(new Partition(spoutConfig, Integer.toString(i)));
+ }
+ logger.info("created partitions, size=" + spoutConfig.getPartitionCount());
+ return partitions;
+ }
+
+ @Override
+ public boolean isReady(long txid) {
+ return true;
}
- logger.info("created partitions, size=" + spoutConfig.getPartitionCount());
- return partitions;
- }
-
- @Override
- public boolean isReady(long txid) {
- return true;
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/ITridentPartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/ITridentPartitionManager.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/ITridentPartitionManager.java
index d1e8b9e..922e334 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/ITridentPartitionManager.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/ITridentPartitionManager.java
@@ -15,21 +15,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
-package org.apache.storm.eventhubs.trident;
-import org.apache.storm.eventhubs.spout.EventDataWrap;
+package org.apache.storm.eventhubs.trident;
import java.util.List;
+import org.apache.storm.eventhubs.spout.EventDataWrap;
public interface ITridentPartitionManager {
- boolean open(String offset);
- void close();
-
- /**
- * receive a batch of messages from EvenHub up to "count" messages
- * @param offset the starting offset
- * @param count max number of messages in this batch
- * @return list of EventData, if failed to receive, return empty list
- */
- public List<EventDataWrap> receiveBatch(String offset, int count);
+ boolean open(String offset);
+
+ void close();
+
+ /**
+ * receive a batch of messages from EvenHub up to "count" messages
+ *
+ * @param offset the starting offset
+ * @param count max number of messages in this batch
+ * @return list of EventData, if failed to receive, return empty list
+ */
+ public List<EventDataWrap> receiveBatch(String offset, int count);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/ITridentPartitionManagerFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/ITridentPartitionManagerFactory.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/ITridentPartitionManagerFactory.java
index 701bd46..15a83cc 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/ITridentPartitionManagerFactory.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/ITridentPartitionManagerFactory.java
@@ -15,12 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.trident;
import java.io.Serializable;
-
import org.apache.storm.eventhubs.spout.IEventHubReceiver;
public interface ITridentPartitionManagerFactory extends Serializable {
- ITridentPartitionManager create(IEventHubReceiver receiver);
+ ITridentPartitionManager create(IEventHubReceiver receiver);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/OpaqueTridentEventHubEmitter.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/OpaqueTridentEventHubEmitter.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/OpaqueTridentEventHubEmitter.java
index 0da421c..4020b21 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/OpaqueTridentEventHubEmitter.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/OpaqueTridentEventHubEmitter.java
@@ -15,54 +15,55 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.trident;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.eventhubs.spout.EventHubSpoutConfig;
import org.apache.storm.eventhubs.spout.IEventHubReceiverFactory;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.spout.IOpaquePartitionedTridentSpout;
import org.apache.storm.trident.topology.TransactionAttempt;
-import java.util.List;
-import java.util.Map;
-
/**
* A thin wrapper of TransactionalTridentEventHubEmitter for OpaqueTridentEventHubSpout
*/
public class OpaqueTridentEventHubEmitter implements IOpaquePartitionedTridentSpout.Emitter<Partitions, Partition, Map> {
- private final TransactionalTridentEventHubEmitter transactionalEmitter;
- public OpaqueTridentEventHubEmitter(EventHubSpoutConfig spoutConfig) {
- transactionalEmitter = new TransactionalTridentEventHubEmitter(spoutConfig);
- }
-
- public OpaqueTridentEventHubEmitter(EventHubSpoutConfig spoutConfig,
- int batchSize,
- ITridentPartitionManagerFactory pmFactory,
- IEventHubReceiverFactory recvFactory) {
- transactionalEmitter = new TransactionalTridentEventHubEmitter(spoutConfig,
- batchSize,
- pmFactory,
- recvFactory);
- }
+ private final TransactionalTridentEventHubEmitter transactionalEmitter;
+
+ public OpaqueTridentEventHubEmitter(EventHubSpoutConfig spoutConfig) {
+ transactionalEmitter = new TransactionalTridentEventHubEmitter(spoutConfig);
+ }
+
+ public OpaqueTridentEventHubEmitter(EventHubSpoutConfig spoutConfig,
+ int batchSize,
+ ITridentPartitionManagerFactory pmFactory,
+ IEventHubReceiverFactory recvFactory) {
+ transactionalEmitter = new TransactionalTridentEventHubEmitter(spoutConfig,
+ batchSize,
+ pmFactory,
+ recvFactory);
+ }
- @Override
- public void close() {
- transactionalEmitter.close();
- }
+ @Override
+ public void close() {
+ transactionalEmitter.close();
+ }
- @Override
- public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector,
- Partition partition, Map meta) {
- return transactionalEmitter.emitPartitionBatchNew(attempt, collector, partition, meta);
- }
+ @Override
+ public Map emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector,
+ Partition partition, Map meta) {
+ return transactionalEmitter.emitPartitionBatchNew(attempt, collector, partition, meta);
+ }
- @Override
- public List<Partition> getOrderedPartitions(Partitions partitions) {
- return transactionalEmitter.getOrderedPartitions(partitions);
- }
+ @Override
+ public List<Partition> getOrderedPartitions(Partitions partitions) {
+ return transactionalEmitter.getOrderedPartitions(partitions);
+ }
- @Override
- public void refreshPartitions(List<Partition> partitionList) {
- transactionalEmitter.refreshPartitions(partitionList);
- }
+ @Override
+ public void refreshPartitions(List<Partition> partitionList) {
+ transactionalEmitter.refreshPartitions(partitionList);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/OpaqueTridentEventHubSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/OpaqueTridentEventHubSpout.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/OpaqueTridentEventHubSpout.java
index 7123304..f062201 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/OpaqueTridentEventHubSpout.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/OpaqueTridentEventHubSpout.java
@@ -15,50 +15,49 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.trident;
import java.util.Map;
-
import org.apache.storm.eventhubs.spout.EventHubSpoutConfig;
import org.apache.storm.eventhubs.spout.IEventDataScheme;
-
import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Fields;
import org.apache.storm.trident.spout.IOpaquePartitionedTridentSpout;
+import org.apache.storm.tuple.Fields;
/**
* Opaque Trident EventHubs Spout
*/
public class OpaqueTridentEventHubSpout implements IOpaquePartitionedTridentSpout<Partitions, Partition, Map> {
- private static final long serialVersionUID = 1L;
- private final IEventDataScheme scheme;
- private final EventHubSpoutConfig spoutConfig;
-
- public OpaqueTridentEventHubSpout(EventHubSpoutConfig config) {
- spoutConfig = config;
- scheme = spoutConfig.getEventDataScheme();
- }
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- return null;
- }
-
- @Override
- public IOpaquePartitionedTridentSpout.Coordinator<Partitions> getCoordinator(
- Map<String, Object> conf, TopologyContext context) {
- return new org.apache.storm.eventhubs.trident.Coordinator(spoutConfig);
- }
-
- @Override
- public IOpaquePartitionedTridentSpout.Emitter<Partitions, Partition, Map> getEmitter(
- Map<String, Object> conf, TopologyContext context) {
- return new OpaqueTridentEventHubEmitter(spoutConfig);
- }
-
- @Override
- public Fields getOutputFields() {
- return scheme.getOutputFields();
- }
+ private static final long serialVersionUID = 1L;
+ private final IEventDataScheme scheme;
+ private final EventHubSpoutConfig spoutConfig;
+
+ public OpaqueTridentEventHubSpout(EventHubSpoutConfig config) {
+ spoutConfig = config;
+ scheme = spoutConfig.getEventDataScheme();
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return null;
+ }
+
+ @Override
+ public IOpaquePartitionedTridentSpout.Coordinator<Partitions> getCoordinator(
+ Map<String, Object> conf, TopologyContext context) {
+ return new org.apache.storm.eventhubs.trident.Coordinator(spoutConfig);
+ }
+
+ @Override
+ public IOpaquePartitionedTridentSpout.Emitter<Partitions, Partition, Map> getEmitter(
+ Map<String, Object> conf, TopologyContext context) {
+ return new OpaqueTridentEventHubEmitter(spoutConfig);
+ }
+
+ @Override
+ public Fields getOutputFields() {
+ return scheme.getOutputFields();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Partition.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Partition.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Partition.java
index 8857eec..44d9d23 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Partition.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Partition.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.trident;
import java.io.Serializable;
@@ -25,15 +26,15 @@ import org.apache.storm.trident.spout.ISpoutPartition;
* Represents an EventHub partition
*/
public class Partition implements ISpoutPartition, Serializable {
- private static final long serialVersionUID = 1L;
- String partitionId;
-
- public Partition(EventHubSpoutConfig config, String partitionId) {
- this.partitionId = partitionId;
- }
-
- @Override
- public String getId() {
- return partitionId;
- }
+ private static final long serialVersionUID = 1L;
+ String partitionId;
+
+ public Partition(EventHubSpoutConfig config, String partitionId) {
+ this.partitionId = partitionId;
+ }
+
+ @Override
+ public String getId() {
+ return partitionId;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Partitions.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Partitions.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Partitions.java
index c3317d9..375f39d 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Partitions.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/Partitions.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.trident;
import java.io.Serializable;
@@ -25,17 +26,18 @@ import java.util.List;
* Represents all EventHub partitions a spout is receiving messages from.
*/
public class Partitions implements Serializable {
- private static final long serialVersionUID = 1L;
- private List<Partition> partitionList;
- public Partitions() {
- partitionList = new ArrayList<Partition>();
- }
-
- public void addPartition(Partition partition) {
- partitionList.add(partition);
- }
-
- public List<Partition> getPartitions() {
- return partitionList;
- }
+ private static final long serialVersionUID = 1L;
+ private List<Partition> partitionList;
+
+ public Partitions() {
+ partitionList = new ArrayList<Partition>();
+ }
+
+ public void addPartition(Partition partition) {
+ partitionList.add(partition);
+ }
+
+ public List<Partition> getPartitions() {
+ return partitionList;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TransactionalTridentEventHubEmitter.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TransactionalTridentEventHubEmitter.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TransactionalTridentEventHubEmitter.java
index d3456e0..8bd7bb1 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TransactionalTridentEventHubEmitter.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TransactionalTridentEventHubEmitter.java
@@ -15,145 +15,149 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.trident;
-import org.apache.storm.eventhubs.spout.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.storm.eventhubs.spout.EventDataWrap;
+import org.apache.storm.eventhubs.spout.EventHubReceiverImpl;
+import org.apache.storm.eventhubs.spout.EventHubSpoutConfig;
+import org.apache.storm.eventhubs.spout.FieldConstants;
+import org.apache.storm.eventhubs.spout.IEventHubReceiver;
+import org.apache.storm.eventhubs.spout.IEventHubReceiverFactory;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.spout.IPartitionedTridentSpout;
import org.apache.storm.trident.topology.TransactionAttempt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
public class TransactionalTridentEventHubEmitter
implements IPartitionedTridentSpout.Emitter<Partitions, Partition, Map<String, Object>> {
- private static final Logger logger = LoggerFactory.getLogger(TransactionalTridentEventHubEmitter.class);
- private final int batchSize;
- private final EventHubSpoutConfig spoutConfig;
- private Map<String, ITridentPartitionManager> pmMap;
- private ITridentPartitionManagerFactory pmFactory;
- private IEventHubReceiverFactory recvFactory;
-
- public TransactionalTridentEventHubEmitter(EventHubSpoutConfig spoutConfig) {
- //use batch size that matches the default credit size
- this(spoutConfig, spoutConfig.getReceiverCredits(), null, null);
- }
-
- public TransactionalTridentEventHubEmitter(final EventHubSpoutConfig spoutConfig,
- int batchSize,
- ITridentPartitionManagerFactory pmFactory,
- IEventHubReceiverFactory recvFactory) {
- this.spoutConfig = spoutConfig;
- this.batchSize = batchSize;
- this.pmFactory = pmFactory;
- this.recvFactory = recvFactory;
- pmMap = new HashMap<String, ITridentPartitionManager>();
- if(this.pmFactory == null) {
- this.pmFactory = new ITridentPartitionManagerFactory() {
- @Override
- public ITridentPartitionManager create(IEventHubReceiver receiver) {
- return new TridentPartitionManager(spoutConfig, receiver);
- }
- };
+ private static final Logger logger = LoggerFactory.getLogger(TransactionalTridentEventHubEmitter.class);
+ private final int batchSize;
+ private final EventHubSpoutConfig spoutConfig;
+ private Map<String, ITridentPartitionManager> pmMap;
+ private ITridentPartitionManagerFactory pmFactory;
+ private IEventHubReceiverFactory recvFactory;
+
+ public TransactionalTridentEventHubEmitter(EventHubSpoutConfig spoutConfig) {
+ //use batch size that matches the default credit size
+ this(spoutConfig, spoutConfig.getReceiverCredits(), null, null);
}
- if(this.recvFactory == null) {
- this.recvFactory = new IEventHubReceiverFactory() {
- @Override
- public IEventHubReceiver create(EventHubSpoutConfig config,
- String partitionId) {
- return new EventHubReceiverImpl(config, partitionId);
+
+ public TransactionalTridentEventHubEmitter(final EventHubSpoutConfig spoutConfig,
+ int batchSize,
+ ITridentPartitionManagerFactory pmFactory,
+ IEventHubReceiverFactory recvFactory) {
+ this.spoutConfig = spoutConfig;
+ this.batchSize = batchSize;
+ this.pmFactory = pmFactory;
+ this.recvFactory = recvFactory;
+ pmMap = new HashMap<String, ITridentPartitionManager>();
+ if (this.pmFactory == null) {
+ this.pmFactory = new ITridentPartitionManagerFactory() {
+ @Override
+ public ITridentPartitionManager create(IEventHubReceiver receiver) {
+ return new TridentPartitionManager(spoutConfig, receiver);
+ }
+ };
+ }
+ if (this.recvFactory == null) {
+ this.recvFactory = new IEventHubReceiverFactory() {
+ @Override
+ public IEventHubReceiver create(EventHubSpoutConfig config,
+ String partitionId) {
+ return new EventHubReceiverImpl(config, partitionId);
+ }
+ };
}
- };
- }
- }
-
- @Override
- public void close() {
- for(ITridentPartitionManager pm: pmMap.values()) {
- pm.close();
- }
- }
-
- /**
- * Check if partition manager for a given partiton is created
- * if not, create it.
- * @param partition
- */
- private ITridentPartitionManager getOrCreatePartitionManager(Partition partition) {
- ITridentPartitionManager pm;
- if(!pmMap.containsKey(partition.getId())) {
- IEventHubReceiver receiver = recvFactory.create(spoutConfig, partition.getId());
- pm = pmFactory.create(receiver);
- pmMap.put(partition.getId(), pm);
}
- else {
- pm = pmMap.get(partition.getId());
+
+ @Override
+ public void close() {
+ for (ITridentPartitionManager pm : pmMap.values()) {
+ pm.close();
+ }
}
- return pm;
- }
-
- @Override
- public void emitPartitionBatch(TransactionAttempt attempt,
- TridentCollector collector, Partition partition, Map<String, Object> meta) {
- String offset = (String)meta.get("offset");
- int count = Integer.parseInt((String)meta.get("count"));
- logger.info("re-emit for partition " + partition.getId() + ", offset=" + offset + ", count=" + count);
- ITridentPartitionManager pm = getOrCreatePartitionManager(partition);
- List<EventDataWrap> listEvents = pm.receiveBatch(offset, count);
- if(listEvents.size() != count) {
- logger.error("failed to refetch eventhub messages, new count=" + listEvents.size());
- return;
+
+ /**
+ * Check if partition manager for a given partiton is created if not, create it.
+ *
+ * @param partition
+ */
+ private ITridentPartitionManager getOrCreatePartitionManager(Partition partition) {
+ ITridentPartitionManager pm;
+ if (!pmMap.containsKey(partition.getId())) {
+ IEventHubReceiver receiver = recvFactory.create(spoutConfig, partition.getId());
+ pm = pmFactory.create(receiver);
+ pmMap.put(partition.getId(), pm);
+ } else {
+ pm = pmMap.get(partition.getId());
+ }
+ return pm;
}
- for(EventDataWrap ed: listEvents) {
- List<Object> tuples =
- spoutConfig.getEventDataScheme().deserialize(ed.getEventData());
- collector.emit(tuples);
+ @Override
+ public void emitPartitionBatch(TransactionAttempt attempt,
+ TridentCollector collector, Partition partition, Map<String, Object> meta) {
+ String offset = (String) meta.get("offset");
+ int count = Integer.parseInt((String) meta.get("count"));
+ logger.info("re-emit for partition " + partition.getId() + ", offset=" + offset + ", count=" + count);
+ ITridentPartitionManager pm = getOrCreatePartitionManager(partition);
+ List<EventDataWrap> listEvents = pm.receiveBatch(offset, count);
+ if (listEvents.size() != count) {
+ logger.error("failed to refetch eventhub messages, new count=" + listEvents.size());
+ return;
+ }
+
+ for (EventDataWrap ed : listEvents) {
+ List<Object> tuples =
+ spoutConfig.getEventDataScheme().deserialize(ed.getEventData());
+ collector.emit(tuples);
+ }
}
- }
-
- @Override
- public Map<String, Object> emitPartitionBatchNew(TransactionAttempt attempt,
- TridentCollector collector, Partition partition, Map<String, Object> meta) {
- ITridentPartitionManager pm = getOrCreatePartitionManager(partition);
- String offset = FieldConstants.DefaultStartingOffset;
- if(meta != null && meta.containsKey("nextOffset")) {
- offset = (String)meta.get("nextOffset");
+
+ @Override
+ public Map<String, Object> emitPartitionBatchNew(TransactionAttempt attempt,
+ TridentCollector collector, Partition partition, Map<String, Object> meta) {
+ ITridentPartitionManager pm = getOrCreatePartitionManager(partition);
+ String offset = FieldConstants.DefaultStartingOffset;
+ if (meta != null && meta.containsKey("nextOffset")) {
+ offset = (String) meta.get("nextOffset");
+ }
+ //logger.info("emit for partition " + partition.getId() + ", offset=" + offset);
+ String nextOffset = offset;
+
+ List<EventDataWrap> listEvents = pm.receiveBatch(offset, batchSize);
+
+ for (EventDataWrap ed : listEvents) {
+ //update nextOffset;
+ nextOffset = ed.getMessageId().getOffset();
+ List<Object> tuples =
+ spoutConfig.getEventDataScheme().deserialize(ed.getEventData());
+ collector.emit(tuples);
+ }
+ //logger.info("emitted new batches: " + listEvents.size());
+
+ Map<String, Object> newMeta = new HashMap<>();
+ newMeta.put("offset", offset);
+ newMeta.put("nextOffset", nextOffset);
+ newMeta.put("count", "" + listEvents.size());
+ return newMeta;
}
- //logger.info("emit for partition " + partition.getId() + ", offset=" + offset);
- String nextOffset = offset;
- List<EventDataWrap> listEvents = pm.receiveBatch(offset, batchSize);
+ @Override
+ public List<Partition> getOrderedPartitions(Partitions partitions) {
+ return partitions.getPartitions();
+ }
- for(EventDataWrap ed: listEvents) {
- //update nextOffset;
- nextOffset = ed.getMessageId().getOffset();
- List<Object> tuples =
- spoutConfig.getEventDataScheme().deserialize(ed.getEventData());
- collector.emit(tuples);
+ @Override
+ public void refreshPartitions(List<Partition> partitionList) {
+ //partition info does not change in EventHub
+ return;
}
- //logger.info("emitted new batches: " + listEvents.size());
-
- Map<String, Object> newMeta = new HashMap<>();
- newMeta.put("offset", offset);
- newMeta.put("nextOffset", nextOffset);
- newMeta.put("count", ""+listEvents.size());
- return newMeta;
- }
-
- @Override
- public List<Partition> getOrderedPartitions(Partitions partitions) {
- return partitions.getPartitions();
- }
-
- @Override
- public void refreshPartitions(List<Partition> partitionList) {
- //partition info does not change in EventHub
- return;
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TransactionalTridentEventHubSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TransactionalTridentEventHubSpout.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TransactionalTridentEventHubSpout.java
index ee3242a..a2eb73d 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TransactionalTridentEventHubSpout.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TransactionalTridentEventHubSpout.java
@@ -15,52 +15,50 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.trident;
import java.util.Map;
-
import org.apache.storm.eventhubs.spout.EventHubSpoutConfig;
import org.apache.storm.eventhubs.spout.IEventDataScheme;
-
import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Fields;
import org.apache.storm.trident.spout.IPartitionedTridentSpout;
-import org.apache.storm.eventhubs.trident.Partition;
+import org.apache.storm.tuple.Fields;
/**
* Transactional Trident EventHub Spout
*/
-public class TransactionalTridentEventHubSpout implements
- IPartitionedTridentSpout<Partitions, Partition, Map<String, Object>> {
- private static final long serialVersionUID = 1L;
- private final IEventDataScheme scheme;
- private final EventHubSpoutConfig spoutConfig;
-
- public TransactionalTridentEventHubSpout(EventHubSpoutConfig config) {
- spoutConfig = config;
- scheme = spoutConfig.getEventDataScheme();
- }
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- return null;
- }
+public class TransactionalTridentEventHubSpout implements
+ IPartitionedTridentSpout<Partitions, Partition, Map<String, Object>> {
+ private static final long serialVersionUID = 1L;
+ private final IEventDataScheme scheme;
+ private final EventHubSpoutConfig spoutConfig;
+
+ public TransactionalTridentEventHubSpout(EventHubSpoutConfig config) {
+ spoutConfig = config;
+ scheme = spoutConfig.getEventDataScheme();
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return null;
+ }
- @Override
- public IPartitionedTridentSpout.Coordinator<Partitions> getCoordinator(
- Map<String, Object> conf, TopologyContext context) {
- return new org.apache.storm.eventhubs.trident.Coordinator(spoutConfig);
- }
+ @Override
+ public IPartitionedTridentSpout.Coordinator<Partitions> getCoordinator(
+ Map<String, Object> conf, TopologyContext context) {
+ return new org.apache.storm.eventhubs.trident.Coordinator(spoutConfig);
+ }
- @Override
- public IPartitionedTridentSpout.Emitter<Partitions, Partition, Map<String, Object>> getEmitter(
- Map<String, Object> conf, TopologyContext context) {
- return new TransactionalTridentEventHubEmitter(spoutConfig);
- }
+ @Override
+ public IPartitionedTridentSpout.Emitter<Partitions, Partition, Map<String, Object>> getEmitter(
+ Map<String, Object> conf, TopologyContext context) {
+ return new TransactionalTridentEventHubEmitter(spoutConfig);
+ }
- @Override
- public Fields getOutputFields() {
- return scheme.getOutputFields();
- }
+ @Override
+ public Fields getOutputFields() {
+ return scheme.getOutputFields();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TridentPartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TridentPartitionManager.java b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TridentPartitionManager.java
index a384667..550e106 100755
--- a/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TridentPartitionManager.java
+++ b/external/storm-eventhubs/src/main/java/org/apache/storm/eventhubs/trident/TridentPartitionManager.java
@@ -15,72 +15,75 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
-package org.apache.storm.eventhubs.trident;
-import org.apache.storm.eventhubs.spout.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+package org.apache.storm.eventhubs.trident;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
+import org.apache.storm.eventhubs.spout.EventDataWrap;
+import org.apache.storm.eventhubs.spout.EventHubException;
+import org.apache.storm.eventhubs.spout.EventHubFilter;
+import org.apache.storm.eventhubs.spout.EventHubSpoutConfig;
+import org.apache.storm.eventhubs.spout.FieldConstants;
+import org.apache.storm.eventhubs.spout.IEventHubReceiver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TridentPartitionManager implements ITridentPartitionManager {
- private static final Logger logger = LoggerFactory.getLogger(TridentPartitionManager.class);
- private final int receiveTimeoutMs = 5000;
- private final IEventHubReceiver receiver;
- private final EventHubSpoutConfig spoutConfig;
- private String lastOffset = FieldConstants.DefaultStartingOffset;
-
- public TridentPartitionManager(EventHubSpoutConfig spoutConfig, IEventHubReceiver receiver) {
- this.receiver = receiver;
- this.spoutConfig = spoutConfig;
- }
-
- @Override
- public boolean open(String offset) {
- try {
- if((offset == null || offset.equals(FieldConstants.DefaultStartingOffset))
- && spoutConfig.getEnqueueTimeFilter() != 0) {
- receiver.open(new EventHubFilter(Instant.ofEpochMilli(spoutConfig.getEnqueueTimeFilter())));
- }
- else {
- receiver.open(new EventHubFilter(offset));
- }
- lastOffset = offset;
- return true;
+ private static final Logger logger = LoggerFactory.getLogger(TridentPartitionManager.class);
+ private final int receiveTimeoutMs = 5000;
+ private final IEventHubReceiver receiver;
+ private final EventHubSpoutConfig spoutConfig;
+ private String lastOffset = FieldConstants.DefaultStartingOffset;
+
+ public TridentPartitionManager(EventHubSpoutConfig spoutConfig, IEventHubReceiver receiver) {
+ this.receiver = receiver;
+ this.spoutConfig = spoutConfig;
}
- catch(EventHubException ex) {
- logger.error("failed to open eventhub receiver: " + ex.getMessage());
- return false;
+
+ @Override
+ public boolean open(String offset) {
+ try {
+ if ((offset == null || offset.equals(FieldConstants.DefaultStartingOffset))
+ && spoutConfig.getEnqueueTimeFilter() != 0) {
+ receiver.open(new EventHubFilter(Instant.ofEpochMilli(spoutConfig.getEnqueueTimeFilter())));
+ } else {
+ receiver.open(new EventHubFilter(offset));
+ }
+ lastOffset = offset;
+ return true;
+ } catch (EventHubException ex) {
+ logger.error("failed to open eventhub receiver: " + ex.getMessage());
+ return false;
+ }
}
- }
-
- @Override
- public void close() {
- receiver.close();
- }
-
- @Override
- public List<EventDataWrap> receiveBatch(String offset, int count) {
- List<EventDataWrap> batch = new ArrayList<EventDataWrap>(count);
- if(!offset.equals(lastOffset) || !receiver.isOpen()) {
- //re-establish connection to eventhub servers using the right offset
- //TBD: might be optimized with cache.
- close();
- if(!open(offset)) {
- return batch;
- }
+
+ @Override
+ public void close() {
+ receiver.close();
}
-
- for(int i=0; i<count; ++i) {
- EventDataWrap ed = receiver.receive();
- if(ed == null) {
- break;
- }
- batch.add(ed);
- lastOffset = ed.getMessageId().getOffset();
+
+ @Override
+ public List<EventDataWrap> receiveBatch(String offset, int count) {
+ List<EventDataWrap> batch = new ArrayList<EventDataWrap>(count);
+ if (!offset.equals(lastOffset) || !receiver.isOpen()) {
+ //re-establish connection to eventhub servers using the right offset
+ //TBD: might be optimized with cache.
+ close();
+ if (!open(offset)) {
+ return batch;
+ }
+ }
+
+ for (int i = 0; i < count; ++i) {
+ EventDataWrap ed = receiver.receive();
+ if (ed == null) {
+ break;
+ }
+ batch.add(ed);
+ lastOffset = ed.getMessageId().getOffset();
+ }
+ return batch;
}
- return batch;
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/AtMostOnceEventCount.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/AtMostOnceEventCount.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/AtMostOnceEventCount.java
index b0dd33a..2c8ef2e 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/AtMostOnceEventCount.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/AtMostOnceEventCount.java
@@ -15,10 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.samples;
import java.io.Serializable;
-
import org.apache.storm.eventhubs.spout.EventHubSpout;
import org.apache.storm.eventhubs.spout.EventHubSpoutConfig;
import org.apache.storm.eventhubs.spout.IEventHubReceiver;
@@ -28,27 +28,27 @@ import org.apache.storm.eventhubs.spout.IStateStore;
import org.apache.storm.eventhubs.spout.SimplePartitionManager;
public class AtMostOnceEventCount extends EventCount implements Serializable {
- @Override
- protected EventHubSpout createEventHubSpout() {
- IPartitionManagerFactory pmFactory = new IPartitionManagerFactory() {
- private static final long serialVersionUID = 1L;
+ public static void main(String[] args) throws Exception {
+ AtMostOnceEventCount scenario = new AtMostOnceEventCount();
+
+ scenario.runScenario(args);
+ }
- @Override
- public IPartitionManager create(EventHubSpoutConfig spoutConfig,
- String partitionId, IStateStore stateStore,
- IEventHubReceiver receiver) {
- return new SimplePartitionManager(spoutConfig, partitionId,
- stateStore, receiver);
- }
- };
- EventHubSpout eventHubSpout = new EventHubSpout(
- spoutConfig, null, pmFactory, null);
- return eventHubSpout;
- }
-
- public static void main(String[] args) throws Exception {
- AtMostOnceEventCount scenario = new AtMostOnceEventCount();
+ @Override
+ protected EventHubSpout createEventHubSpout() {
+ IPartitionManagerFactory pmFactory = new IPartitionManagerFactory() {
+ private static final long serialVersionUID = 1L;
- scenario.runScenario(args);
- }
+ @Override
+ public IPartitionManager create(EventHubSpoutConfig spoutConfig,
+ String partitionId, IStateStore stateStore,
+ IEventHubReceiver receiver) {
+ return new SimplePartitionManager(spoutConfig, partitionId,
+ stateStore, receiver);
+ }
+ };
+ EventHubSpout eventHubSpout = new EventHubSpout(
+ spoutConfig, null, pmFactory, null);
+ return eventHubSpout;
+ }
}
[55/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-hive
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-hive
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5fc4e9f0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5fc4e9f0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5fc4e9f0
Branch: refs/heads/master
Commit: 5fc4e9f0bdf7a58852f7c27a1f8049e2bb3776a5
Parents: 0e409ec
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 23:19:45 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 02:32:40 2018 -0400
----------------------------------------------------------------------
external/storm-hive/pom.xml | 2 +-
.../org/apache/storm/hive/bolt/HiveBolt.java | 139 ++++-----
.../bolt/mapper/DelimitedRecordHiveMapper.java | 60 ++--
.../storm/hive/bolt/mapper/HiveMapper.java | 30 +-
.../hive/bolt/mapper/JsonRecordHiveMapper.java | 61 ++--
.../apache/storm/hive/common/HiveOptions.java | 24 +-
.../org/apache/storm/hive/common/HiveUtils.java | 60 ++--
.../apache/storm/hive/common/HiveWriter.java | 255 ++++++++--------
.../apache/storm/hive/trident/HiveState.java | 124 ++++----
.../storm/hive/trident/HiveStateFactory.java | 31 +-
.../apache/storm/hive/trident/HiveUpdater.java | 23 +-
.../apache/storm/hive/bolt/HiveSetupUtil.java | 143 +++++----
.../apache/storm/hive/bolt/TestHiveBolt.java | 302 +++++++++----------
.../storm/hive/common/TestHiveWriter.java | 205 ++++++-------
14 files changed, 683 insertions(+), 776 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-hive/pom.xml b/external/storm-hive/pom.xml
index 4637e7f..eb790f7 100644
--- a/external/storm-hive/pom.xml
+++ b/external/storm-hive/pom.xml
@@ -202,7 +202,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>259</maxAllowedViolations>
+ <maxAllowedViolations>58</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/HiveBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/HiveBolt.java b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/HiveBolt.java
index 7a76888..cfabbd6 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/HiveBolt.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/HiveBolt.java
@@ -1,59 +1,52 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hive.bolt;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.hcatalog.streaming.HiveEndPoint;
import org.apache.hive.hcatalog.streaming.SerializationError;
import org.apache.hive.hcatalog.streaming.StreamingException;
+import org.apache.storm.Config;
+import org.apache.storm.hive.common.HiveOptions;
+import org.apache.storm.hive.common.HiveUtils;
+import org.apache.storm.hive.common.HiveWriter;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.topology.base.BaseRichBolt;
+import org.apache.storm.tuple.Tuple;
import org.apache.storm.utils.BatchHelper;
import org.apache.storm.utils.TupleUtils;
-import org.apache.storm.Config;
-import org.apache.storm.hive.common.HiveWriter;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-import org.apache.storm.hive.common.HiveOptions;
-import org.apache.storm.hive.common.HiveUtils;
-import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.Map.Entry;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.List;
-import java.io.IOException;
-
public class HiveBolt extends BaseRichBolt {
private static final Logger LOG = LoggerFactory.getLogger(HiveBolt.class);
+ @VisibleForTesting
+ Map<HiveEndPoint, HiveWriter> allWriters;
private OutputCollector collector;
private HiveOptions options;
private ExecutorService callTimeoutPool;
@@ -63,36 +56,33 @@ public class HiveBolt extends BaseRichBolt {
private BatchHelper batchHelper;
private boolean tokenAuthEnabled;
- @VisibleForTesting
- Map<HiveEndPoint, HiveWriter> allWriters;
-
public HiveBolt(HiveOptions options) {
this.options = options;
}
@Override
- public void prepare(Map<String, Object> conf, TopologyContext topologyContext, OutputCollector collector) {
+ public void prepare(Map<String, Object> conf, TopologyContext topologyContext, OutputCollector collector) {
try {
tokenAuthEnabled = HiveUtils.isTokenAuthEnabled(conf);
try {
ugi = HiveUtils.authenticate(tokenAuthEnabled, options.getKerberosKeytab(), options.getKerberosPrincipal());
- } catch(HiveUtils.AuthenticationFailed ex) {
+ } catch (HiveUtils.AuthenticationFailed ex) {
LOG.error("Hive kerberos authentication failed " + ex.getMessage(), ex);
throw new IllegalArgumentException(ex);
}
this.collector = collector;
this.batchHelper = new BatchHelper(options.getBatchSize(), collector);
- allWriters = new ConcurrentHashMap<HiveEndPoint,HiveWriter>();
+ allWriters = new ConcurrentHashMap<HiveEndPoint, HiveWriter>();
String timeoutName = "hive-bolt-%d";
this.callTimeoutPool = Executors.newFixedThreadPool(1,
- new ThreadFactoryBuilder().setNameFormat(timeoutName).build());
+ new ThreadFactoryBuilder().setNameFormat(timeoutName).build());
sendHeartBeat.set(true);
heartBeatTimer = new Timer();
setupHeartBeatTimer();
- } catch(Exception e) {
+ } catch (Exception e) {
LOG.warn("unable to make connection to hive ", e);
}
}
@@ -108,7 +98,7 @@ public class HiveBolt extends BaseRichBolt {
batchHelper.addBatch(tuple);
}
- if(batchHelper.shouldFlush()) {
+ if (batchHelper.shouldFlush()) {
flushAllWriters(true);
LOG.info("acknowledging tuples after writers flushed ");
batchHelper.ack();
@@ -116,11 +106,11 @@ public class HiveBolt extends BaseRichBolt {
if (TupleUtils.isTick(tuple)) {
retireIdleWriters();
}
- } catch(SerializationError se) {
+ } catch (SerializationError se) {
LOG.info("Serialization exception occurred, tuple is acknowledged but not written to Hive.", tuple);
this.collector.reportError(se);
collector.ack(tuple);
- } catch(Exception e) {
+ } catch (Exception e) {
batchHelper.fail(e);
abortAndCloseWriters();
}
@@ -147,13 +137,13 @@ public class HiveBolt extends BaseRichBolt {
}
}
- ExecutorService toShutdown[] = {callTimeoutPool};
+ ExecutorService toShutdown[] = { callTimeoutPool };
for (ExecutorService execService : toShutdown) {
execService.shutdown();
try {
while (!execService.isTerminated()) {
execService.awaitTermination(
- options.getCallTimeOut(), TimeUnit.MILLISECONDS);
+ options.getCallTimeOut(), TimeUnit.MILLISECONDS);
}
} catch (InterruptedException ex) {
LOG.warn("shutdown interrupted on " + execService, ex);
@@ -168,31 +158,33 @@ public class HiveBolt extends BaseRichBolt {
@Override
public Map<String, Object> getComponentConfiguration() {
Map<String, Object> conf = super.getComponentConfiguration();
- if (conf == null)
+ if (conf == null) {
conf = new Config();
+ }
- if (options.getTickTupleInterval() > 0)
+ if (options.getTickTupleInterval() > 0) {
conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, options.getTickTupleInterval());
+ }
return conf;
}
private void setupHeartBeatTimer() {
- if(options.getHeartBeatInterval()>0) {
+ if (options.getHeartBeatInterval() > 0) {
heartBeatTimer.schedule(new TimerTask() {
- @Override
- public void run() {
- try {
- if (sendHeartBeat.get()) {
- LOG.debug("Start sending heartbeat on all writers");
- sendHeartBeatOnAllWriters();
- setupHeartBeatTimer();
- }
- } catch (Exception e) {
- LOG.warn("Failed to heartbeat on HiveWriter ", e);
+ @Override
+ public void run() {
+ try {
+ if (sendHeartBeat.get()) {
+ LOG.debug("Start sending heartbeat on all writers");
+ sendHeartBeatOnAllWriters();
+ setupHeartBeatTimer();
}
+ } catch (Exception e) {
+ LOG.warn("Failed to heartbeat on HiveWriter ", e);
}
- }, options.getHeartBeatInterval() * 1000);
+ }
+ }, options.getHeartBeatInterval() * 1000);
}
}
@@ -204,7 +196,7 @@ public class HiveBolt extends BaseRichBolt {
void flushAllWriters(boolean rollToNext)
throws HiveWriter.CommitFailure, HiveWriter.TxnBatchFailure, HiveWriter.TxnFailure, InterruptedException {
- for(HiveWriter writer: allWriters.values()) {
+ for (HiveWriter writer : allWriters.values()) {
writer.flush(rollToNext);
}
}
@@ -213,7 +205,7 @@ public class HiveBolt extends BaseRichBolt {
try {
abortAllWriters();
closeAllWriters();
- } catch(Exception ie) {
+ } catch (Exception ie) {
LOG.warn("unable to close hive connections. ", ie);
}
}
@@ -222,11 +214,11 @@ public class HiveBolt extends BaseRichBolt {
* Abort current Txn on all writers
*/
private void abortAllWriters() throws InterruptedException, StreamingException, HiveWriter.TxnBatchFailure {
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
+ for (Entry<HiveEndPoint, HiveWriter> entry : allWriters.entrySet()) {
try {
entry.getValue().abort();
} catch (Exception e) {
- LOG.error("Failed to abort hive transaction batch, HiveEndPoint " + entry.getValue() +" due to exception ", e);
+ LOG.error("Failed to abort hive transaction batch, HiveEndPoint " + entry.getValue() + " due to exception ", e);
}
}
}
@@ -236,10 +228,10 @@ public class HiveBolt extends BaseRichBolt {
*/
private void closeAllWriters() {
//1) Retire writers
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
+ for (Entry<HiveEndPoint, HiveWriter> entry : allWriters.entrySet()) {
try {
entry.getValue().close();
- } catch(Exception e) {
+ } catch (Exception e) {
LOG.warn("unable to close writers. ", e);
}
}
@@ -251,14 +243,15 @@ public class HiveBolt extends BaseRichBolt {
HiveWriter getOrCreateWriter(HiveEndPoint endPoint)
throws HiveWriter.ConnectFailure, InterruptedException {
try {
- HiveWriter writer = allWriters.get( endPoint );
+ HiveWriter writer = allWriters.get(endPoint);
if (writer == null) {
LOG.debug("Creating Writer to Hive end point : " + endPoint);
writer = HiveUtils.makeHiveWriter(endPoint, callTimeoutPool, ugi, options, tokenAuthEnabled);
if (allWriters.size() > (options.getMaxOpenConnections() - 1)) {
- LOG.info("cached HiveEndPoint size {} exceeded maxOpenConnections {} ", allWriters.size(), options.getMaxOpenConnections());
+ LOG.info("cached HiveEndPoint size {} exceeded maxOpenConnections {} ", allWriters.size(),
+ options.getMaxOpenConnections());
int retired = retireIdleWriters();
- if(retired==0) {
+ if (retired == 0) {
retireEldestWriter();
}
}
@@ -279,7 +272,7 @@ public class HiveBolt extends BaseRichBolt {
LOG.info("Attempting close eldest writers");
long oldestTimeStamp = System.currentTimeMillis();
HiveEndPoint eldest = null;
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
+ for (Entry<HiveEndPoint, HiveWriter> entry : allWriters.entrySet()) {
if (entry.getValue().getLastUsed() < oldestTimeStamp) {
eldest = entry.getKey();
oldestTimeStamp = entry.getValue().getLastUsed();
@@ -308,8 +301,8 @@ public class HiveBolt extends BaseRichBolt {
long now = System.currentTimeMillis();
//1) Find retirement candidates
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
- if(now - entry.getValue().getLastUsed() > options.getIdleTimeout()) {
+ for (Entry<HiveEndPoint, HiveWriter> entry : allWriters.entrySet()) {
+ if (now - entry.getValue().getLastUsed() > options.getIdleTimeout()) {
++count;
retire(entry.getKey());
}
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/DelimitedRecordHiveMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/DelimitedRecordHiveMapper.java b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/DelimitedRecordHiveMapper.java
index 48cd5ff..d67ec67 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/DelimitedRecordHiveMapper.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/DelimitedRecordHiveMapper.java
@@ -1,40 +1,34 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hive.bolt.mapper;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.trident.tuple.TridentTuple;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
import org.apache.hive.hcatalog.streaming.DelimitedInputWriter;
import org.apache.hive.hcatalog.streaming.HiveEndPoint;
import org.apache.hive.hcatalog.streaming.RecordWriter;
import org.apache.hive.hcatalog.streaming.StreamingException;
import org.apache.hive.hcatalog.streaming.TransactionBatch;
-
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Date;
-import java.text.SimpleDateFormat;
-import java.io.IOException;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class DelimitedRecordHiveMapper implements HiveMapper {
private static final Logger LOG = LoggerFactory.getLogger(DelimitedRecordHiveMapper.class);
@@ -62,7 +56,7 @@ public class DelimitedRecordHiveMapper implements HiveMapper {
return this;
}
- public DelimitedRecordHiveMapper withFieldDelimiter(String delimiter){
+ public DelimitedRecordHiveMapper withFieldDelimiter(String delimiter) {
this.fieldDelimiter = delimiter;
return this;
}
@@ -76,7 +70,7 @@ public class DelimitedRecordHiveMapper implements HiveMapper {
@Override
public RecordWriter createRecordWriter(HiveEndPoint endPoint)
throws StreamingException, IOException, ClassNotFoundException {
- return new DelimitedInputWriter(columnNames, fieldDelimiter,endPoint);
+ return new DelimitedInputWriter(columnNames, fieldDelimiter, endPoint);
}
@Override
@@ -88,8 +82,8 @@ public class DelimitedRecordHiveMapper implements HiveMapper {
@Override
public List<String> mapPartitions(Tuple tuple) {
List<String> partitionList = new ArrayList<String>();
- if(this.partitionFields != null) {
- for(String field: this.partitionFields) {
+ if (this.partitionFields != null) {
+ for (String field : this.partitionFields) {
partitionList.add(tuple.getStringByField(field));
}
}
@@ -102,8 +96,8 @@ public class DelimitedRecordHiveMapper implements HiveMapper {
@Override
public byte[] mapRecord(Tuple tuple) {
StringBuilder builder = new StringBuilder();
- if(this.columnFields != null) {
- for(String field: this.columnFields) {
+ if (this.columnFields != null) {
+ for (String field : this.columnFields) {
builder.append(tuple.getValueByField(field));
builder.append(fieldDelimiter);
}
@@ -114,8 +108,8 @@ public class DelimitedRecordHiveMapper implements HiveMapper {
@Override
public List<String> mapPartitions(TridentTuple tuple) {
List<String> partitionList = new ArrayList<String>();
- if(this.partitionFields != null) {
- for(String field: this.partitionFields) {
+ if (this.partitionFields != null) {
+ for (String field : this.partitionFields) {
partitionList.add(tuple.getStringByField(field));
}
}
@@ -128,8 +122,8 @@ public class DelimitedRecordHiveMapper implements HiveMapper {
@Override
public byte[] mapRecord(TridentTuple tuple) {
StringBuilder builder = new StringBuilder();
- if(this.columnFields != null) {
- for(String field: this.columnFields) {
+ if (this.columnFields != null) {
+ for (String field : this.columnFields) {
builder.append(tuple.getValueByField(field));
builder.append(fieldDelimiter);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/HiveMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/HiveMapper.java b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/HiveMapper.java
index a7a79fb..cc29392 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/HiveMapper.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/HiveMapper.java
@@ -1,33 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hive.bolt.mapper;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.trident.tuple.TridentTuple;
+import java.io.IOException;
+import java.io.Serializable;
import java.util.List;
import org.apache.hive.hcatalog.streaming.HiveEndPoint;
import org.apache.hive.hcatalog.streaming.RecordWriter;
-import org.apache.hive.hcatalog.streaming.TransactionBatch;
import org.apache.hive.hcatalog.streaming.StreamingException;
-import java.io.Serializable;
-
-import java.io.IOException;
+import org.apache.hive.hcatalog.streaming.TransactionBatch;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Tuple;
/**
* Maps a <code>org.apache.storm.tuple.Tupe</code> object
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/JsonRecordHiveMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/JsonRecordHiveMapper.java b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/JsonRecordHiveMapper.java
index 0028b87..3a43b7c 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/JsonRecordHiveMapper.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/bolt/mapper/JsonRecordHiveMapper.java
@@ -1,41 +1,34 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hive.bolt.mapper;
+package org.apache.storm.hive.bolt.mapper;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.trident.tuple.TridentTuple;
-import org.apache.storm.tuple.Tuple;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
import org.apache.hive.hcatalog.streaming.HiveEndPoint;
import org.apache.hive.hcatalog.streaming.RecordWriter;
import org.apache.hive.hcatalog.streaming.StreamingException;
import org.apache.hive.hcatalog.streaming.StrictJsonWriter;
import org.apache.hive.hcatalog.streaming.TransactionBatch;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
import org.json.simple.JSONObject;
-
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Date;
-import java.text.SimpleDateFormat;
-import java.io.IOException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class JsonRecordHiveMapper implements HiveMapper {
private static final Logger LOG = LoggerFactory.getLogger(JsonRecordHiveMapper.class);
@@ -78,8 +71,8 @@ public class JsonRecordHiveMapper implements HiveMapper {
@Override
public List<String> mapPartitions(Tuple tuple) {
List<String> partitionList = new ArrayList<String>();
- if(this.partitionFields != null) {
- for(String field: this.partitionFields) {
+ if (this.partitionFields != null) {
+ for (String field : this.partitionFields) {
partitionList.add(tuple.getStringByField(field));
}
}
@@ -92,9 +85,9 @@ public class JsonRecordHiveMapper implements HiveMapper {
@Override
public byte[] mapRecord(Tuple tuple) {
JSONObject obj = new JSONObject();
- if(this.columnFields != null) {
- for(String field: this.columnFields) {
- obj.put(field,tuple.getValueByField(field));
+ if (this.columnFields != null) {
+ for (String field : this.columnFields) {
+ obj.put(field, tuple.getValueByField(field));
}
}
return obj.toJSONString().getBytes();
@@ -103,8 +96,8 @@ public class JsonRecordHiveMapper implements HiveMapper {
@Override
public List<String> mapPartitions(TridentTuple tuple) {
List<String> partitionList = new ArrayList<String>();
- if(this.partitionFields != null) {
- for(String field: this.partitionFields) {
+ if (this.partitionFields != null) {
+ for (String field : this.partitionFields) {
partitionList.add(tuple.getStringByField(field));
}
}
@@ -117,9 +110,9 @@ public class JsonRecordHiveMapper implements HiveMapper {
@Override
public byte[] mapRecord(TridentTuple tuple) {
JSONObject obj = new JSONObject();
- if(this.columnFields != null) {
- for(String field: this.columnFields) {
- obj.put(field,tuple.getValueByField(field));
+ if (this.columnFields != null) {
+ for (String field : this.columnFields) {
+ obj.put(field, tuple.getValueByField(field));
}
}
return obj.toJSONString().getBytes();
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveOptions.java b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveOptions.java
index e80f5d7..4a91da1 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveOptions.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveOptions.java
@@ -1,25 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hive.common;
import java.io.Serializable;
-
import org.apache.storm.hive.bolt.mapper.HiveMapper;
public class HiveOptions implements Serializable {
@@ -43,15 +36,14 @@ public class HiveOptions implements Serializable {
protected String kerberosKeytab;
protected Integer tickTupleInterval = DEFAULT_TICK_TUPLE_INTERVAL_SECS;
- public HiveOptions(String metaStoreURI,String databaseName,String tableName,HiveMapper mapper) {
+ public HiveOptions(String metaStoreURI, String databaseName, String tableName, HiveMapper mapper) {
this.metaStoreURI = metaStoreURI;
this.databaseName = databaseName;
this.tableName = tableName;
this.mapper = mapper;
}
- public HiveOptions withTickTupleInterval(Integer tickInterval)
- {
+ public HiveOptions withTickTupleInterval(Integer tickInterval) {
this.tickTupleInterval = tickInterval;
return this;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveUtils.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveUtils.java b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveUtils.java
index 26beee0..7681f91 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveUtils.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveUtils.java
@@ -1,23 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hive.common;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.hcatalog.streaming.ConnectionError;
@@ -26,12 +25,6 @@ import org.apache.storm.hive.security.AutoHive;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-
import static org.apache.storm.Config.TOPOLOGY_AUTO_CREDENTIALS;
public class HiveUtils {
@@ -44,16 +37,19 @@ public class HiveUtils {
return new HiveEndPoint(options.getMetaStoreURI(), options.getDatabaseName(), options.getTableName(), partitionVals);
}
- public static HiveWriter makeHiveWriter(HiveEndPoint endPoint, ExecutorService callTimeoutPool, UserGroupInformation ugi, HiveOptions options, boolean tokenAuthEnabled)
- throws HiveWriter.ConnectFailure, InterruptedException {
+ public static HiveWriter makeHiveWriter(HiveEndPoint endPoint, ExecutorService callTimeoutPool, UserGroupInformation ugi,
+ HiveOptions options, boolean tokenAuthEnabled)
+ throws HiveWriter.ConnectFailure, InterruptedException {
return new HiveWriter(endPoint, options.getTxnsPerBatch(), options.getAutoCreatePartitions(),
- options.getCallTimeOut(), callTimeoutPool, options.getMapper(), ugi, tokenAuthEnabled);
+ options.getCallTimeOut(), callTimeoutPool, options.getMapper(), ugi, tokenAuthEnabled);
}
- public static synchronized UserGroupInformation authenticate(boolean isTokenAuthEnabled, String keytab, String principal) throws AuthenticationFailed {
+ public static synchronized UserGroupInformation authenticate(boolean isTokenAuthEnabled, String keytab, String principal) throws
+ AuthenticationFailed {
- if (isTokenAuthEnabled)
+ if (isTokenAuthEnabled) {
return getCurrentUser(principal);
+ }
boolean kerberosEnabled = false;
@@ -70,7 +66,7 @@ public class HiveUtils {
if (!(kfile.isFile() && kfile.canRead())) {
throw new IllegalArgumentException("The keyTab file: " + keytab + " is nonexistent or can't read. "
- + "Please specify a readable keytab file for Kerberos auth.");
+ + "Please specify a readable keytab file for Kerberos auth.");
}
try {
@@ -91,12 +87,6 @@ public class HiveUtils {
}
- public static class AuthenticationFailed extends Exception {
- public AuthenticationFailed(String reason, Exception cause) {
- super("Kerberos Authentication Failed. " + reason, cause);
- }
- }
-
public static void logAllHiveEndPoints(Map<HiveEndPoint, HiveWriter> allWriters) {
for (Map.Entry<HiveEndPoint, HiveWriter> entry : allWriters.entrySet()) {
LOG.info("cached writers {} ", entry.getValue());
@@ -104,10 +94,10 @@ public class HiveUtils {
}
public static boolean isTokenAuthEnabled(Map<String, Object> conf) {
- return conf.get(TOPOLOGY_AUTO_CREDENTIALS) != null && (((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoHive.class.getName()));
+ return conf.get(TOPOLOGY_AUTO_CREDENTIALS) != null &&
+ (((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoHive.class.getName()));
}
-
private static UserGroupInformation getCurrentUser(String principal) throws AuthenticationFailed {
try {
return UserGroupInformation.getCurrentUser();
@@ -115,4 +105,10 @@ public class HiveUtils {
throw new AuthenticationFailed("Login failed for principal " + principal, e);
}
}
+
+ public static class AuthenticationFailed extends Exception {
+ public AuthenticationFailed(String reason, Exception cause) {
+ super("Kerberos Authentication Failed. " + reason, cause);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java
index e9a30fe..bc0ee75 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/common/HiveWriter.java
@@ -1,23 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hive.common;
+import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.concurrent.Callable;
@@ -26,14 +21,17 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.hcatalog.streaming.*;
+import org.apache.hive.hcatalog.streaming.HiveEndPoint;
+import org.apache.hive.hcatalog.streaming.RecordWriter;
+import org.apache.hive.hcatalog.streaming.SerializationError;
+import org.apache.hive.hcatalog.streaming.StreamingConnection;
+import org.apache.hive.hcatalog.streaming.StreamingException;
+import org.apache.hive.hcatalog.streaming.StreamingIOFailure;
+import org.apache.hive.hcatalog.streaming.TransactionBatch;
import org.apache.storm.hive.bolt.mapper.HiveMapper;
import org.apache.storm.tuple.Tuple;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,9 +48,9 @@ public class HiveWriter {
private final ExecutorService callTimeoutPool;
private final long callTimeout;
private final Object txnBatchLock = new Object();
+ protected boolean closed; // flag indicating HiveWriter was closed
private TransactionBatch txnBatch;
private long lastUsed; // time of last flush on this writer
- protected boolean closed; // flag indicating HiveWriter was closed
private boolean autoCreatePartitions;
private UserGroupInformation ugi;
private int totalRecords = 0;
@@ -74,37 +72,52 @@ public class HiveWriter {
this.txnBatch = nextTxnBatch(recordWriter);
this.closed = false;
this.lastUsed = System.currentTimeMillis();
- } catch(InterruptedException e) {
+ } catch (InterruptedException e) {
throw e;
- } catch(RuntimeException e) {
+ } catch (RuntimeException e) {
throw e;
- } catch(Exception e) {
+ } catch (Exception e) {
throw new ConnectFailure(endPoint, e);
}
}
- public RecordWriter getRecordWriter(final HiveMapper mapper, final boolean tokenAuthEnabled) throws Exception {
- if (!tokenAuthEnabled)
- return mapper.createRecordWriter(endPoint);
+ /**
+ * If the current thread has been interrupted, then throws an
+ * exception.
+ * @throws InterruptedException
+ */
+ private static void checkAndThrowInterruptedException()
+ throws InterruptedException {
+ if (Thread.currentThread().interrupted()) {
+ throw new InterruptedException("Timed out before Hive call was made. "
+ + "Your callTimeout might be set too low or Hive calls are "
+ + "taking too long.");
+ }
+ }
+
+ public RecordWriter getRecordWriter(final HiveMapper mapper, final boolean tokenAuthEnabled) throws Exception {
+ if (!tokenAuthEnabled) {
+ return mapper.createRecordWriter(endPoint);
+ }
try {
- return ugi.doAs (
- new PrivilegedExceptionAction<RecordWriter>() {
- @Override
- public RecordWriter run() throws StreamingException, IOException, ClassNotFoundException {
- return mapper.createRecordWriter(endPoint);
- }
+ return ugi.doAs(
+ new PrivilegedExceptionAction<RecordWriter>() {
+ @Override
+ public RecordWriter run() throws StreamingException, IOException, ClassNotFoundException {
+ return mapper.createRecordWriter(endPoint);
}
+ }
);
} catch (Exception e) {
throw new ConnectFailure(endPoint, e);
}
}
-
- private HiveConf createHiveConf(String metaStoreURI, boolean tokenAuthEnabled) {
- if (!tokenAuthEnabled)
+ private HiveConf createHiveConf(String metaStoreURI, boolean tokenAuthEnabled) {
+ if (!tokenAuthEnabled) {
return null;
+ }
HiveConf hcatConf = new HiveConf();
hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreURI);
@@ -114,9 +127,9 @@ public class HiveWriter {
@Override
public String toString() {
- return "{ "
- + "endPoint = " + endPoint.toString()
- + ", TransactionBatch = " + txnBatch.toString() + " }";
+ return "{ "
+ + "endPoint = " + endPoint.toString()
+ + ", TransactionBatch = " + txnBatch.toString() + " }";
}
/**
@@ -135,18 +148,18 @@ public class HiveWriter {
try {
LOG.debug("Writing event to {}", endPoint);
callWithTimeout(new CallRunner<Void>() {
- @Override
- public Void call() throws StreamingException, InterruptedException {
- txnBatch.write(record);
- totalRecords++;
- return null;
- }
- });
- } catch(SerializationError se) {
+ @Override
+ public Void call() throws StreamingException, InterruptedException {
+ txnBatch.write(record);
+ totalRecords++;
+ return null;
+ }
+ });
+ } catch (SerializationError se) {
throw new SerializationError(endPoint.toString() + " SerializationError", se);
- } catch(StreamingException e) {
+ } catch (StreamingException e) {
throw new WriteFailure(endPoint, txnBatch.getCurrentTxnId(), e);
- } catch(TimeoutException e) {
+ } catch (TimeoutException e) {
throw new WriteFailure(endPoint, txnBatch.getCurrentTxnId(), e);
}
}
@@ -161,13 +174,13 @@ public class HiveWriter {
// if there are no records do not call flush
if (totalRecords <= 0) return;
try {
- synchronized(txnBatchLock) {
+ synchronized (txnBatchLock) {
commitTxn();
nextTxn(rollToNext);
totalRecords = 0;
lastUsed = System.currentTimeMillis();
}
- } catch(StreamingException e) {
+ } catch (StreamingException e) {
throw new TxnFailure(txnBatch, e);
}
}
@@ -177,24 +190,24 @@ public class HiveWriter {
*/
public void heartBeat() throws InterruptedException {
// 1) schedule the heartbeat on one thread in pool
- synchronized(txnBatchLock) {
+ synchronized (txnBatchLock) {
try {
callWithTimeout(new CallRunner<Void>() {
- @Override
- public Void call() throws Exception {
- try {
- LOG.info("Sending heartbeat on batch " + txnBatch);
- txnBatch.heartbeat();
- } catch (StreamingException e) {
- LOG.warn("Heartbeat error on batch " + txnBatch, e);
- }
- return null;
+ @Override
+ public Void call() throws Exception {
+ try {
+ LOG.info("Sending heartbeat on batch " + txnBatch);
+ txnBatch.heartbeat();
+ } catch (StreamingException e) {
+ LOG.warn("Heartbeat error on batch " + txnBatch, e);
}
- });
+ return null;
+ }
+ });
} catch (InterruptedException e) {
throw e;
} catch (Exception e) {
- LOG.warn("Unable to send heartbeat on Txn Batch " + txnBatch, e);
+ LOG.warn("Unable to send heartbeat on Txn Batch " + txnBatch, e);
// Suppressing exceptions as we don't care for errors on heartbeats
}
}
@@ -212,10 +225,11 @@ public class HiveWriter {
* Flush and Close current transactionBatch.
*/
public void flushAndClose() throws TxnBatchFailure, TxnFailure, CommitFailure,
- IOException, InterruptedException {
+ IOException, InterruptedException {
flush(false);
close();
}
+
/**
* Close the Transaction Batch and connection
* @throws IOException
@@ -231,28 +245,28 @@ public class HiveWriter {
LOG.info("Closing connection to end point : {}", endPoint);
try {
callWithTimeout(new CallRunner<Void>() {
- @Override
- public Void call() throws Exception {
- connection.close(); // could block
- return null;
- }
- });
- } catch(Exception e) {
+ @Override
+ public Void call() throws Exception {
+ connection.close(); // could block
+ return null;
+ }
+ });
+ } catch (Exception e) {
LOG.warn("Error closing connection to EndPoint : " + endPoint, e);
// Suppressing exceptions as we don't care for errors on connection close
}
}
private void commitTxn() throws CommitFailure, InterruptedException {
- LOG.debug("Committing Txn id {} to {}", txnBatch.getCurrentTxnId() , endPoint);
+ LOG.debug("Committing Txn id {} to {}", txnBatch.getCurrentTxnId(), endPoint);
try {
callWithTimeout(new CallRunner<Void>() {
- @Override
- public Void call() throws Exception {
- txnBatch.commit(); // could block
- return null;
- }
- });
+ @Override
+ public Void call() throws Exception {
+ txnBatch.commit(); // could block
+ return null;
+ }
+ });
} catch (StreamingException e) {
throw new CommitFailure(endPoint, txnBatch.getCurrentTxnId(), e);
} catch (TimeoutException e) {
@@ -264,15 +278,16 @@ public class HiveWriter {
StreamingConnection newConnection(final UserGroupInformation ugi, final boolean tokenAuthEnabled)
throws InterruptedException, ConnectFailure {
try {
- return callWithTimeout(new CallRunner<StreamingConnection>() {
- @Override
- public StreamingConnection call() throws Exception {
- return endPoint.newConnection(autoCreatePartitions, createHiveConf(endPoint.metaStoreUri, tokenAuthEnabled) , ugi); // could block
- }
- });
- } catch(StreamingException e) {
+ return callWithTimeout(new CallRunner<StreamingConnection>() {
+ @Override
+ public StreamingConnection call() throws Exception {
+ return endPoint
+ .newConnection(autoCreatePartitions, createHiveConf(endPoint.metaStoreUri, tokenAuthEnabled), ugi); // could block
+ }
+ });
+ } catch (StreamingException e) {
throw new ConnectFailure(endPoint, e);
- } catch(TimeoutException e) {
+ } catch (TimeoutException e) {
throw new ConnectFailure(endPoint, e);
}
}
@@ -290,30 +305,30 @@ public class HiveWriter {
});
batch.beginNextTransaction();
LOG.debug("Acquired {}. Switching to first txn", batch);
- } catch(TimeoutException e) {
+ } catch (TimeoutException e) {
throw new TxnBatchFailure(endPoint, e);
- } catch(StreamingException e) {
+ } catch (StreamingException e) {
throw new TxnBatchFailure(endPoint, e);
}
return batch;
}
- private void closeTxnBatch() throws InterruptedException {
+ private void closeTxnBatch() throws InterruptedException {
try {
LOG.debug("Closing Txn Batch {}", txnBatch);
callWithTimeout(new CallRunner<Void>() {
- @Override
- public Void call() throws Exception {
- if(txnBatch != null) {
- txnBatch.close(); // could block
- }
- return null;
+ @Override
+ public Void call() throws Exception {
+ if (txnBatch != null) {
+ txnBatch.close(); // could block
}
- });
- } catch(InterruptedException e) {
+ return null;
+ }
+ });
+ } catch (InterruptedException e) {
throw e;
- } catch(Exception e) {
- LOG.warn("Error closing txn batch "+ txnBatch, e);
+ } catch (Exception e) {
+ LOG.warn("Error closing txn batch " + txnBatch, e);
}
}
@@ -322,13 +337,12 @@ public class HiveWriter {
* @throws StreamingException if could not get new Transaction Batch, or switch to next Txn
*/
public void abort() throws StreamingException, TxnBatchFailure, InterruptedException {
- synchronized(txnBatchLock) {
+ synchronized (txnBatchLock) {
abortTxn();
nextTxn(true); // roll to next
}
}
-
/**
* Aborts current Txn in the txnBatch.
*/
@@ -336,12 +350,12 @@ public class HiveWriter {
LOG.info("Aborting Txn id {} on End Point {}", txnBatch.getCurrentTxnId(), endPoint);
try {
callWithTimeout(new CallRunner<Void>() {
- @Override
- public Void call() throws StreamingException, InterruptedException {
- txnBatch.abort(); // could block
- return null;
- }
- });
+ @Override
+ public Void call() throws StreamingException, InterruptedException {
+ txnBatch.abort(); // could block
+ return null;
+ }
+ });
} catch (InterruptedException e) {
throw e;
} catch (TimeoutException e) {
@@ -352,40 +366,25 @@ public class HiveWriter {
}
}
-
/**
* if there are remainingTransactions in current txnBatch, begins nextTransactions
* otherwise creates new txnBatch.
* @param rollToNext
*/
private void nextTxn(boolean rollToNext) throws StreamingException, InterruptedException, TxnBatchFailure {
- if(txnBatch.remainingTransactions() == 0) {
+ if (txnBatch.remainingTransactions() == 0) {
closeTxnBatch();
txnBatch = null;
- if(rollToNext) {
+ if (rollToNext) {
txnBatch = nextTxnBatch(recordWriter);
}
- } else if(rollToNext) {
+ } else if (rollToNext) {
LOG.debug("Switching to next Txn for {}", endPoint);
txnBatch.beginNextTransaction(); // does not block
}
}
/**
- * If the current thread has been interrupted, then throws an
- * exception.
- * @throws InterruptedException
- */
- private static void checkAndThrowInterruptedException()
- throws InterruptedException {
- if (Thread.currentThread().interrupted()) {
- throw new InterruptedException("Timed out before Hive call was made. "
- + "Your callTimeout might be set too low or Hive calls are "
- + "taking too long.");
- }
- }
-
- /**
* Execute the callable on a separate thread and wait for the completion
* for the specified amount of time in milliseconds. In case of timeout
* cancel the callable and throw an IOException
@@ -393,11 +392,11 @@ public class HiveWriter {
private <T> T callWithTimeout(final CallRunner<T> callRunner)
throws TimeoutException, StreamingException, InterruptedException {
Future<T> future = callTimeoutPool.submit(new Callable<T>() {
- @Override
- public T call() throws Exception {
- return callRunner.call();
- }
- });
+ @Override
+ public T call() throws Exception {
+ return callRunner.call();
+ }
+ });
try {
if (callTimeout > 0) {
return future.get(callTimeout, TimeUnit.MILLISECONDS);
@@ -431,7 +430,7 @@ public class HiveWriter {
private byte[] generateRecord(Tuple tuple) {
StringBuilder buf = new StringBuilder();
- for (Object o: tuple.getValues()) {
+ for (Object o : tuple.getValues()) {
buf.append(o);
buf.append(",");
}
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveState.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveState.java b/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveState.java
index a7685f0..a698e24 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveState.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveState.java
@@ -1,50 +1,43 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hive.trident;
-import org.apache.storm.trident.operation.TridentCollector;
-import org.apache.storm.trident.state.State;
-import org.apache.storm.trident.tuple.TridentTuple;
-import org.apache.storm.task.IMetricsContext;
-import org.apache.storm.topology.FailedException;
-import org.apache.storm.hive.common.HiveWriter;
-import org.apache.hive.hcatalog.streaming.*;
-import org.apache.storm.hive.common.HiveOptions;
-import org.apache.storm.hive.common.HiveUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.Map.Entry;
import java.util.Timer;
import java.util.TimerTask;
-import java.util.Map.Entry;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hive.hcatalog.streaming.HiveEndPoint;
+import org.apache.hive.hcatalog.streaming.StreamingException;
+import org.apache.storm.hive.common.HiveOptions;
+import org.apache.storm.hive.common.HiveUtils;
+import org.apache.storm.hive.common.HiveWriter;
+import org.apache.storm.task.IMetricsContext;
+import org.apache.storm.topology.FailedException;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.state.State;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HiveState implements State {
private static final Logger LOG = LoggerFactory.getLogger(HiveState.class);
@@ -73,24 +66,24 @@ public class HiveState implements State {
public void commit(Long txId) {
}
- public void prepare(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ public void prepare(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
try {
tokenAuthEnabled = HiveUtils.isTokenAuthEnabled(conf);
try {
ugi = HiveUtils.authenticate(tokenAuthEnabled, options.getKerberosKeytab(), options.getKerberosPrincipal());
- } catch(HiveUtils.AuthenticationFailed ex) {
+ } catch (HiveUtils.AuthenticationFailed ex) {
LOG.error("Hive kerberos authentication failed " + ex.getMessage(), ex);
throw new IllegalArgumentException(ex);
}
- allWriters = new ConcurrentHashMap<HiveEndPoint,HiveWriter>();
+ allWriters = new ConcurrentHashMap<HiveEndPoint, HiveWriter>();
String timeoutName = "hive-bolt-%d";
this.callTimeoutPool = Executors.newFixedThreadPool(1,
new ThreadFactoryBuilder().setNameFormat(timeoutName).build());
- heartBeatTimer= new Timer();
+ heartBeatTimer = new Timer();
setupHeartBeatTimer();
- } catch(Exception e) {
- LOG.warn("unable to make connection to hive ",e);
+ } catch (Exception e) {
+ LOG.warn("unable to make connection to hive ", e);
}
}
@@ -99,7 +92,7 @@ public class HiveState implements State {
writeTuples(tuples);
} catch (Exception e) {
abortAndCloseWriters();
- LOG.warn("hive streaming failed.",e);
+ LOG.warn("hive streaming failed.", e);
throw new FailedException(e);
}
}
@@ -112,7 +105,7 @@ public class HiveState implements State {
HiveWriter writer = getOrCreateWriter(endPoint);
writer.write(options.getMapper().mapRecord(tuple));
currentBatchSize++;
- if(currentBatchSize >= options.getBatchSize()) {
+ if (currentBatchSize >= options.getBatchSize()) {
flushAllWriters();
currentBatchSize = 0;
}
@@ -124,7 +117,7 @@ public class HiveState implements State {
sendHeartBeat = false;
abortAllWriters();
closeAllWriters();
- } catch(Exception ie) {
+ } catch (Exception ie) {
LOG.warn("unable to close hive connections. ", ie);
}
}
@@ -133,7 +126,7 @@ public class HiveState implements State {
* Abort current Txn on all writers
*/
private void abortAllWriters() throws InterruptedException, StreamingException, HiveWriter.TxnBatchFailure {
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
+ for (Entry<HiveEndPoint, HiveWriter> entry : allWriters.entrySet()) {
entry.getValue().abort();
}
}
@@ -145,7 +138,7 @@ public class HiveState implements State {
*/
private void closeAllWriters() throws InterruptedException, IOException {
//1) Retire writers
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
+ for (Entry<HiveEndPoint, HiveWriter> entry : allWriters.entrySet()) {
entry.getValue().close();
}
//2) Clear cache
@@ -153,27 +146,27 @@ public class HiveState implements State {
}
private void setupHeartBeatTimer() {
- if(options.getHeartBeatInterval()>0) {
+ if (options.getHeartBeatInterval() > 0) {
heartBeatTimer.schedule(new TimerTask() {
- @Override
- public void run() {
- try {
- if (sendHeartBeat) {
- LOG.debug("Start sending heartbeat on all writers");
- sendHeartBeatOnAllWriters();
- setupHeartBeatTimer();
- }
- } catch (Exception e) {
- LOG.warn("Failed to heartbeat on HiveWriter ", e);
+ @Override
+ public void run() {
+ try {
+ if (sendHeartBeat) {
+ LOG.debug("Start sending heartbeat on all writers");
+ sendHeartBeatOnAllWriters();
+ setupHeartBeatTimer();
}
+ } catch (Exception e) {
+ LOG.warn("Failed to heartbeat on HiveWriter ", e);
}
- }, options.getHeartBeatInterval() * 1000);
+ }
+ }, options.getHeartBeatInterval() * 1000);
}
}
private void flushAllWriters()
throws HiveWriter.CommitFailure, HiveWriter.TxnBatchFailure, HiveWriter.TxnFailure, InterruptedException {
- for(HiveWriter writer: allWriters.values()) {
+ for (HiveWriter writer : allWriters.values()) {
writer.flush(true);
}
}
@@ -187,13 +180,13 @@ public class HiveState implements State {
private HiveWriter getOrCreateWriter(HiveEndPoint endPoint)
throws HiveWriter.ConnectFailure, InterruptedException {
try {
- HiveWriter writer = allWriters.get( endPoint );
- if( writer == null ) {
+ HiveWriter writer = allWriters.get(endPoint);
+ if (writer == null) {
LOG.info("Creating Writer to Hive end point : " + endPoint);
writer = HiveUtils.makeHiveWriter(endPoint, callTimeoutPool, ugi, options, tokenAuthEnabled);
- if(allWriters.size() > (options.getMaxOpenConnections() - 1)){
+ if (allWriters.size() > (options.getMaxOpenConnections() - 1)) {
int retired = retireIdleWriters();
- if(retired==0) {
+ if (retired == 0) {
retireEldestWriter();
}
}
@@ -208,15 +201,14 @@ public class HiveState implements State {
}
-
/**
* Locate writer that has not been used for longest time and retire it
*/
private void retireEldestWriter() {
long oldestTimeStamp = System.currentTimeMillis();
HiveEndPoint eldest = null;
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
- if(entry.getValue().getLastUsed() < oldestTimeStamp) {
+ for (Entry<HiveEndPoint, HiveWriter> entry : allWriters.entrySet()) {
+ if (entry.getValue().getLastUsed() < oldestTimeStamp) {
eldest = entry.getKey();
oldestTimeStamp = entry.getValue().getLastUsed();
}
@@ -244,19 +236,19 @@ public class HiveState implements State {
ArrayList<HiveEndPoint> retirees = new ArrayList<HiveEndPoint>();
//1) Find retirement candidates
- for (Entry<HiveEndPoint,HiveWriter> entry : allWriters.entrySet()) {
- if(now - entry.getValue().getLastUsed() > options.getIdleTimeout()) {
+ for (Entry<HiveEndPoint, HiveWriter> entry : allWriters.entrySet()) {
+ if (now - entry.getValue().getLastUsed() > options.getIdleTimeout()) {
++count;
retirees.add(entry.getKey());
}
}
//2) Retire them
- for(HiveEndPoint ep : retirees) {
+ for (HiveEndPoint ep : retirees) {
try {
LOG.info("Closing idle Writer to Hive end point : {}", ep);
allWriters.remove(ep).flushAndClose();
} catch (IOException e) {
- LOG.warn("Failed to close writer for end point: {}. Error: "+ ep, e);
+ LOG.warn("Failed to close writer for end point: {}. Error: " + ep, e);
} catch (InterruptedException e) {
LOG.warn("Interrupted when attempting to close writer for end point: " + ep, e);
Thread.currentThread().interrupt();
@@ -285,13 +277,13 @@ public class HiveState implements State {
}
}
- ExecutorService toShutdown[] = {callTimeoutPool};
+ ExecutorService toShutdown[] = { callTimeoutPool };
for (ExecutorService execService : toShutdown) {
execService.shutdown();
try {
while (!execService.isTerminated()) {
execService.awaitTermination(
- options.getCallTimeOut(), TimeUnit.MILLISECONDS);
+ options.getCallTimeOut(), TimeUnit.MILLISECONDS);
}
} catch (InterruptedException ex) {
LOG.warn("shutdown interrupted on " + execService, ex);
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveStateFactory.java b/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveStateFactory.java
index 6659825..68722d1 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveStateFactory.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveStateFactory.java
@@ -1,40 +1,33 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hive.trident;
+import java.util.Map;
+import org.apache.storm.hive.common.HiveOptions;
import org.apache.storm.task.IMetricsContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.state.StateFactory;
-import org.apache.storm.hive.common.HiveOptions;
-
-import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HiveStateFactory implements StateFactory {
private static final Logger LOG = LoggerFactory.getLogger(HiveStateFactory.class);
private HiveOptions options;
- public HiveStateFactory(){}
+ public HiveStateFactory() {}
- public HiveStateFactory withOptions(HiveOptions options){
+ public HiveStateFactory withOptions(HiveOptions options) {
this.options = options;
return this;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveUpdater.java b/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveUpdater.java
index 062f7fb..36479d9 100644
--- a/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveUpdater.java
+++ b/external/storm-hive/src/main/java/org/apache/storm/hive/trident/HiveUpdater.java
@@ -1,30 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hive.trident;
+import java.util.List;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseStateUpdater;
import org.apache.storm.trident.tuple.TridentTuple;
-import java.util.List;
-
-public class HiveUpdater extends BaseStateUpdater<HiveState>{
+public class HiveUpdater extends BaseStateUpdater<HiveState> {
@Override
public void updateState(HiveState state, List<TridentTuple> tuples, TridentCollector collector) {
state.updateState(tuples, collector);
http://git-wip-us.apache.org/repos/asf/storm/blob/5fc4e9f0/external/storm-hive/src/test/java/org/apache/storm/hive/bolt/HiveSetupUtil.java
----------------------------------------------------------------------
diff --git a/external/storm-hive/src/test/java/org/apache/storm/hive/bolt/HiveSetupUtil.java b/external/storm-hive/src/test/java/org/apache/storm/hive/bolt/HiveSetupUtil.java
index d492819..6646390 100644
--- a/external/storm-hive/src/test/java/org/apache/storm/hive/bolt/HiveSetupUtil.java
+++ b/external/storm-hive/src/test/java/org/apache/storm/hive/bolt/HiveSetupUtil.java
@@ -1,23 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hive.bolt;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
@@ -42,57 +45,7 @@ import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.thrift.TException;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
public class HiveSetupUtil {
- public static class RawFileSystem extends RawLocalFileSystem {
- private static final URI NAME;
- static {
- try {
- NAME = new URI("raw:///");
- } catch (URISyntaxException se) {
- throw new IllegalArgumentException("bad uri", se);
- }
- }
-
- @Override
- public URI getUri() {
- return NAME;
- }
-
- @Override
- public FileStatus getFileStatus(Path path) throws IOException {
- File file = pathToFile(path);
- if (!file.exists()) {
- throw new FileNotFoundException("Can't find " + path);
- }
- // get close enough
- short mod = 0;
- if (file.canRead()) {
- mod |= 0444;
- }
- if (file.canWrite()) {
- mod |= 0200;
- }
- if (file.canExecute()) {
- mod |= 0111;
- }
- ShimLoader.getHadoopShims();
- return new FileStatus(file.length(), file.isDirectory(), 1, 1024,
- file.lastModified(), file.lastModified(),
- FsPermission.createImmutable(mod), "owen", "users", path);
- }
- }
-
private final static String txnMgr = "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager";
public static HiveConf getHiveConf() {
@@ -126,7 +79,7 @@ public class HiveSetupUtil {
sd.setCols(getTableColumns(colNames, colTypes));
sd.setNumBuckets(1);
sd.setLocation(dbLocation + Path.SEPARATOR + tableName);
- if(partNames!=null && partNames.length!=0) {
+ if (partNames != null && partNames.length != 0) {
tbl.setPartitionKeys(getPartitionKeys(partNames));
}
@@ -146,10 +99,10 @@ public class HiveSetupUtil {
tbl.setParameters(tableParams);
client.createTable(tbl);
try {
- if(partVals!=null && partVals.size() > 0) {
+ if (partVals != null && partVals.size() > 0) {
addPartition(client, tbl, partVals);
}
- } catch(AlreadyExistsException e) {
+ } catch (AlreadyExistsException e) {
}
} finally {
client.close();
@@ -170,7 +123,7 @@ public class HiveSetupUtil {
}
private static void addPartition(IMetaStoreClient client, Table tbl
- , List<String> partValues)
+ , List<String> partValues)
throws IOException, TException {
Partition part = new Partition();
part.setDbName(tbl.getDbName());
@@ -183,17 +136,17 @@ public class HiveSetupUtil {
}
private static String makePartPath(List<FieldSchema> partKeys, List<String> partVals) {
- if(partKeys.size()!=partVals.size()) {
+ if (partKeys.size() != partVals.size()) {
throw new IllegalArgumentException("Partition values:" + partVals +
- ", does not match the partition Keys in table :" + partKeys );
+ ", does not match the partition Keys in table :" + partKeys);
}
- StringBuffer buff = new StringBuffer(partKeys.size()*20);
- int i=0;
- for(FieldSchema schema : partKeys) {
+ StringBuffer buff = new StringBuffer(partKeys.size() * 20);
+ int i = 0;
+ for (FieldSchema schema : partKeys) {
buff.append(schema.getName());
buff.append("=");
buff.append(partVals.get(i));
- if(i!=partKeys.size()-1) {
+ if (i != partKeys.size() - 1) {
buff.append(Path.SEPARATOR);
}
++i;
@@ -203,7 +156,7 @@ public class HiveSetupUtil {
private static List<FieldSchema> getTableColumns(String[] colNames, String[] colTypes) {
List<FieldSchema> fields = new ArrayList<FieldSchema>();
- for (int i=0; i<colNames.length; ++i) {
+ for (int i = 0; i < colNames.length; ++i) {
fields.add(new FieldSchema(colNames[i], colTypes[i], ""));
}
return fields;
@@ -211,10 +164,50 @@ public class HiveSetupUtil {
private static List<FieldSchema> getPartitionKeys(String[] partNames) {
List<FieldSchema> fields = new ArrayList<FieldSchema>();
- for (int i=0; i < partNames.length; ++i) {
- fields.add(new FieldSchema(partNames[i], serdeConstants.STRING_TYPE_NAME, ""));
+ for (int i = 0; i < partNames.length; ++i) {
+ fields.add(new FieldSchema(partNames[i], serdeConstants.STRING_TYPE_NAME, ""));
}
return fields;
}
+ public static class RawFileSystem extends RawLocalFileSystem {
+ private static final URI NAME;
+
+ static {
+ try {
+ NAME = new URI("raw:///");
+ } catch (URISyntaxException se) {
+ throw new IllegalArgumentException("bad uri", se);
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return NAME;
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path path) throws IOException {
+ File file = pathToFile(path);
+ if (!file.exists()) {
+ throw new FileNotFoundException("Can't find " + path);
+ }
+ // get close enough
+ short mod = 0;
+ if (file.canRead()) {
+ mod |= 0444;
+ }
+ if (file.canWrite()) {
+ mod |= 0200;
+ }
+ if (file.canExecute()) {
+ mod |= 0111;
+ }
+ ShimLoader.getHadoopShims();
+ return new FileStatus(file.length(), file.isDirectory(), 1, 1024,
+ file.lastModified(), file.lastModified(),
+ FsPermission.createImmutable(mod), "owen", "users", path);
+ }
+ }
+
}
[62/63] [abbrv] storm git commit: Fix checkstyle issues
Posted by bo...@apache.org.
Fix checkstyle issues
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d9ca88ac
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d9ca88ac
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d9ca88ac
Branch: refs/heads/master
Commit: d9ca88ac53cf512ee19241f9f0d4eec5d37ab514
Parents: 99c62d4
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Wed Apr 25 17:06:35 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Wed Apr 25 17:06:35 2018 -0400
----------------------------------------------------------------------
.../storm/cluster/PaceMakerStateStorage.java | 2 +-
.../apache/storm/pacemaker/PacemakerClient.java | 17 +++++++----------
.../storm/pacemaker/PacemakerClientPool.java | 4 ++--
3 files changed, 10 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d9ca88ac/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java b/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java
index 47a3501..54d3f91 100644
--- a/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java
+++ b/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java
@@ -129,7 +129,7 @@ public class PaceMakerStateStorage implements IStateStorage {
}
LOG.debug("Successful set_worker_hb");
break;
- } catch (HBExecutionException|PacemakerConnectionException e) {
+ } catch (HBExecutionException | PacemakerConnectionException e) {
if (retry <= 0) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/d9ca88ac/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java
index 1f03854..46b7683 100644
--- a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java
+++ b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClient.java
@@ -52,7 +52,7 @@ public class PacemakerClient implements ISaslClient {
private LinkedBlockingQueue<Integer> availableMessageSlots;
private ThriftNettyClientCodec.AuthMethod authMethod;
private static final int maxRetries = 10;
-
+
private StormBoundedExponentialBackoffRetry backoff = new StormBoundedExponentialBackoffRetry(100, 5000, 20);
private int retryTimes = 0;
@@ -106,8 +106,7 @@ public class PacemakerClient implements ISaslClient {
ThreadFactory bossFactory = new NettyRenameThreadFactory("client-boss");
ThreadFactory workerFactory = new NettyRenameThreadFactory("client-worker");
NioClientSocketChannelFactory factory =
- new NioClientSocketChannelFactory(Executors.newCachedThreadPool(bossFactory),
- Executors.newCachedThreadPool(workerFactory));
+ new NioClientSocketChannelFactory(Executors.newCachedThreadPool(bossFactory), Executors.newCachedThreadPool(workerFactory));
bootstrap = new ClientBootstrap(factory);
bootstrap.setOption("tcpNoDelay", true);
bootstrap.setOption("sendBufferSize", 5242880);
@@ -116,8 +115,7 @@ public class PacemakerClient implements ISaslClient {
remote_addr = new InetSocketAddress(host, port);
int thriftMessageMaxSize = (Integer) config.get(Config.PACEMAKER_THRIFT_MESSAGE_SIZE_MAX);
ChannelPipelineFactory pipelineFactory =
- new ThriftNettyClientCodec(this, config, authMethod, host, thriftMessageMaxSize)
- .pipelineFactory();
+ new ThriftNettyClientCodec(this, config, authMethod, host, thriftMessageMaxSize).pipelineFactory();
bootstrap.setPipelineFactory(pipelineFactory);
bootstrap.connect(remote_addr);
}
@@ -242,11 +240,10 @@ public class PacemakerClient implements ISaslClient {
public void reconnect() {
final PacemakerClient client = this;
timer.schedule(new TimerTask() {
- public void run() {
- client.doReconnect();
- }
- },
- backoff.getSleepTimeMs(retryTimes++, 0));
+ public void run() {
+ client.doReconnect();
+ }
+ }, backoff.getSleepTimeMs(retryTimes++, 0));
ready.set(false);
setupMessaging();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/d9ca88ac/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java
index 6721ecb..bfed672 100644
--- a/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java
+++ b/storm-client/src/jvm/org/apache/storm/pacemaker/PacemakerClientPool.java
@@ -46,7 +46,7 @@ public class PacemakerClientPool {
servers = new ConcurrentLinkedQueue<>();
}
}
-
+
public HBMessage send(HBMessage m) throws PacemakerConnectionException, InterruptedException {
try {
return getWriteClient().send(m);
@@ -59,7 +59,7 @@ public class PacemakerClientPool {
public List<HBMessage> sendAll(HBMessage m) throws PacemakerConnectionException, InterruptedException {
List<HBMessage> responses = new ArrayList<HBMessage>();
LOG.debug("Using servers: {}", servers);
- for(String s : servers) {
+ for (String s : servers) {
try {
HBMessage response = getClientForServer(s).send(m);
responses.add(response);
[63/63] [abbrv] storm git commit: Merge branch 'fix-style' of
https://github.com/kishorvpatil/incubator-storm into STORM-3037
Posted by bo...@apache.org.
Merge branch 'fix-style' of https://github.com/kishorvpatil/incubator-storm into STORM-3037
STORM-3037: Lowering CheckStyle Violations across all modules
This closes #2641
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4137328b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4137328b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4137328b
Branch: refs/heads/master
Commit: 4137328b75c06771f84414c3c2113e2d1c757c08
Parents: 7673253 d9ca88a
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Thu Apr 26 15:27:49 2018 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Thu Apr 26 15:27:49 2018 -0500
----------------------------------------------------------------------
examples/storm-hbase-examples/pom.xml | 5 +-
.../storm/hbase/topology/LookupWordCount.java | 28 +-
.../hbase/topology/PersistentWordCount.java | 36 +-
.../storm/hbase/topology/TotalWordCounter.java | 31 +-
.../storm/hbase/topology/WordCountClient.java | 21 +-
.../hbase/topology/WordCountValueMapper.java | 43 +-
.../storm/hbase/topology/WordCounter.java | 22 +-
.../apache/storm/hbase/topology/WordSpout.java | 28 +-
.../storm/hbase/trident/PrintFunction.java | 28 +-
.../storm/hbase/trident/WordCountTrident.java | 57 +-
examples/storm-hdfs-examples/pom.xml | 2 +-
.../storm/hdfs/bolt/HdfsFileTopology.java | 54 +-
.../storm/hdfs/bolt/SequenceFileTopology.java | 58 +-
.../storm/hdfs/spout/HdfsSpoutTopology.java | 307 ++-
.../storm/hdfs/trident/FixedBatchSpout.java | 29 +-
.../storm/hdfs/trident/TridentFileTopology.java | 30 +-
.../hdfs/trident/TridentSequenceTopology.java | 30 +-
examples/storm-perf/pom.xml | 2 +-
.../perf/ConstSpoutIdBoltNullBoltTopo.java | 12 +-
.../storm/perf/ConstSpoutNullBoltTopo.java | 4 +-
.../storm/perf/FileReadWordCountTopo.java | 33 +-
.../storm/perf/HdfsSpoutNullBoltTopo.java | 3 +-
.../org/apache/storm/perf/JCQueuePerfTest.java | 31 +-
.../org/apache/storm/perf/JCToolsPerfTest.java | 27 +-
.../apache/storm/perf/KafkaClientHdfsTopo.java | 8 +-
.../perf/KafkaClientSpoutNullBoltTopo.java | 15 +-
.../apache/storm/perf/LowThroughputTopo.java | 9 +-
.../storm/perf/SimplifiedWordCountTopo.java | 1 -
.../storm/perf/StrGenSpoutHdfsBoltTopo.java | 3 +-
.../org/apache/storm/perf/ThroughputMeter.java | 16 +-
.../org/apache/storm/perf/bolt/CountBolt.java | 1 -
.../org/apache/storm/perf/bolt/DevNullBolt.java | 1 -
.../java/org/apache/storm/perf/bolt/IdBolt.java | 1 -
.../storm/perf/bolt/SplitSentenceBolt.java | 1 -
.../org/apache/storm/perf/spout/ConstSpout.java | 1 -
.../apache/storm/perf/spout/FileReadSpout.java | 1 -
.../apache/storm/perf/spout/StringGenSpout.java | 27 +-
.../apache/storm/perf/spout/WordGenSpout.java | 62 +-
.../storm/perf/utils/BasicMetricsCollector.java | 23 +-
.../org/apache/storm/perf/utils/Helper.java | 7 +-
.../apache/storm/perf/utils/IdentityBolt.java | 3 +-
.../apache/storm/perf/utils/MetricsSample.java | 3 +-
examples/storm-starter/pom.xml | 2 +-
.../apache/storm/starter/AnchoredWordCount.java | 72 +-
.../apache/storm/starter/BasicDRPCTopology.java | 49 +-
.../starter/BlobStoreAPIWordCountTopology.java | 276 ++-
.../storm/starter/ExclamationTopology.java | 84 +-
.../storm/starter/FastWordCountTopology.java | 300 +--
.../storm/starter/InOrderDeliveryTest.java | 261 +--
.../apache/storm/starter/JoinBoltExample.java | 39 +-
.../apache/storm/starter/LambdaTopology.java | 24 +-
.../org/apache/storm/starter/ManualDRPC.java | 51 +-
.../storm/starter/MultipleLoggerTopology.java | 112 +-
.../starter/PersistentWindowingTopology.java | 99 +-
.../org/apache/storm/starter/ReachTopology.java | 76 +-
.../starter/ResourceAwareExampleTopology.java | 113 +-
.../apache/storm/starter/RollingTopWords.java | 111 +-
.../apache/storm/starter/SingleJoinExample.java | 26 +-
.../storm/starter/SkewedRollingTopWords.java | 113 +-
.../storm/starter/SlidingTupleTsTopology.java | 30 +-
.../storm/starter/SlidingWindowTopology.java | 62 +-
.../apache/storm/starter/StatefulTopology.java | 44 +-
.../starter/StatefulWindowingTopology.java | 46 +-
.../storm/starter/TransactionalGlobalCount.java | 65 +-
.../storm/starter/TransactionalWords.java | 102 +-
.../apache/storm/starter/WordCountTopology.java | 115 +-
.../storm/starter/WordCountTopologyNode.java | 61 +-
.../storm/starter/bolt/AbstractRankerBolt.java | 135 +-
.../starter/bolt/IntermediateRankingsBolt.java | 61 +-
.../apache/storm/starter/bolt/PrinterBolt.java | 33 +-
.../storm/starter/bolt/RollingCountAggBolt.java | 92 +-
.../storm/starter/bolt/RollingCountBolt.java | 169 +-
.../storm/starter/bolt/SingleJoinBolt.java | 167 +-
.../starter/bolt/SlidingWindowSumBolt.java | 48 +-
.../storm/starter/bolt/TotalRankingsBolt.java | 63 +-
.../storm/starter/spout/RandomIntegerSpout.java | 24 +-
.../spout/RandomNumberGeneratorSpout.java | 35 +-
.../starter/spout/RandomSentenceSpout.java | 120 +-
.../storm/starter/streams/AggregateExample.java | 33 +-
.../storm/starter/streams/BranchExample.java | 45 +-
.../streams/GroupByKeyAndWindowExample.java | 62 +-
.../storm/starter/streams/JoinExample.java | 48 +-
.../starter/streams/StateQueryExample.java | 53 +-
.../starter/streams/StatefulWordCount.java | 59 +-
.../starter/streams/TypedTupleExample.java | 19 +-
.../starter/streams/WindowedWordCount.java | 70 +-
.../storm/starter/streams/WordCountToBolt.java | 51 +-
.../tools/NthLastModifiedTimeTracker.java | 75 +-
.../apache/storm/starter/tools/Rankable.java | 35 +-
.../starter/tools/RankableObjectWithFields.java | 224 +-
.../apache/storm/starter/tools/Rankings.java | 238 +-
.../starter/tools/SlidingWindowCounter.java | 101 +-
.../storm/starter/tools/SlotBasedCounter.java | 144 +-
.../starter/trident/DebugMemoryMapState.java | 7 +-
.../TridentHBaseWindowingStoreTopology.java | 46 +-
.../starter/trident/TridentMapExample.java | 50 +-
.../trident/TridentMinMaxOfDevicesTopology.java | 85 +-
.../TridentMinMaxOfVehiclesTopology.java | 83 +-
.../storm/starter/trident/TridentReach.java | 95 +-
.../TridentWindowingInmemoryStoreTopology.java | 43 +-
.../storm/starter/trident/TridentWordCount.java | 58 +-
.../bolt/IntermediateRankingsBoltTest.java | 244 ++-
.../starter/bolt/RollingCountBoltTest.java | 178 +-
.../starter/bolt/TotalRankingsBoltTest.java | 246 ++-
.../tools/NthLastModifiedTimeTrackerTest.java | 158 +-
.../tools/RankableObjectWithFieldsTest.java | 489 +++--
.../storm/starter/tools/RankingsTest.java | 668 +++---
.../starter/tools/SlidingWindowCounterTest.java | 175 +-
.../starter/tools/SlotBasedCounterTest.java | 320 ++-
external/storm-cassandra/pom.xml | 2 +-
.../AbstractExecutionResultHandler.java | 34 +-
.../cassandra/BaseExecutionResultHandler.java | 32 +-
.../storm/cassandra/CassandraContext.java | 28 +-
.../cassandra/DynamicStatementBuilder.java | 38 +-
.../storm/cassandra/ExecutionResultHandler.java | 27 +-
.../storm/cassandra/Murmur3StreamGrouping.java | 87 +-
.../storm/cassandra/bolt/BaseCassandraBolt.java | 42 +-
.../bolt/BatchCassandraWriterBolt.java | 80 +-
.../cassandra/bolt/CassandraWriterBolt.java | 34 +-
.../cassandra/bolt/GroupingBatchBuilder.java | 29 +-
.../bolt/PairBatchStatementTuples.java | 23 +-
.../cassandra/bolt/PairStatementTuple.java | 26 +-
.../storm/cassandra/client/CassandraConf.java | 191 +-
.../storm/cassandra/client/ClusterFactory.java | 46 +-
.../storm/cassandra/client/SimpleClient.java | 20 +-
.../cassandra/client/SimpleClientProvider.java | 20 +-
.../cassandra/client/impl/DefaultClient.java | 27 +-
.../cassandra/context/BaseBeanFactory.java | 25 +-
.../storm/cassandra/context/BeanFactory.java | 20 +-
.../storm/cassandra/context/WorkerCtx.java | 32 +-
.../storm/cassandra/executor/AsyncExecutor.java | 98 +-
.../executor/AsyncExecutorProvider.java | 22 +-
.../cassandra/executor/AsyncResultHandler.java | 27 +-
.../executor/AsyncResultSetHandler.java | 23 +-
.../executor/ExecutionResultCollector.java | 36 +-
.../executor/impl/BatchAsyncResultHandler.java | 32 +-
.../executor/impl/SingleAsyncResultHandler.java | 30 +-
.../query/AyncCQLResultSetValuesMapper.java | 25 +-
.../query/BaseCQLStatementTupleMapper.java | 23 +-
.../query/CQLResultSetValuesMapper.java | 25 +-
.../cassandra/query/CQLStatementBuilder.java | 20 +-
.../query/CQLStatementTupleMapper.java | 25 +-
.../apache/storm/cassandra/query/Column.java | 34 +-
.../storm/cassandra/query/ContextQuery.java | 23 +-
.../apache/storm/cassandra/query/CqlMapper.java | 29 +-
.../cassandra/query/ObjectMapperOperation.java | 19 +-
.../builder/BoundCQLStatementMapperBuilder.java | 33 +-
.../ObjectMapperCqlStatementMapperBuilder.java | 19 +-
.../SimpleCQLStatementMapperBuilder.java | 31 +-
.../impl/BatchCQLStatementTupleMapper.java | 29 +-
.../impl/BoundCQLStatementTupleMapper.java | 48 +-
.../impl/ObjectMapperCqlStatementMapper.java | 27 +-
.../query/impl/PreparedStatementBinder.java | 40 +-
.../query/impl/RoutingKeyGenerator.java | 25 +-
.../query/impl/SimpleCQLStatementMapper.java | 36 +-
.../cassandra/query/selector/FieldSelector.java | 25 +-
.../trident/state/CassandraBackingMap.java | 49 +-
.../trident/state/CassandraMapStateFactory.java | 29 +-
.../cassandra/trident/state/CassandraQuery.java | 25 +-
.../cassandra/trident/state/CassandraState.java | 91 +-
.../trident/state/CassandraStateFactory.java | 28 +-
.../trident/state/CassandraStateUpdater.java | 23 +-
.../trident/state/MapStateFactoryBuilder.java | 85 +-
.../state/NonTransactionalTupleStateMapper.java | 23 +-
.../trident/state/OpaqueTupleStateMapper.java | 31 +-
.../trident/state/SerializedStateMapper.java | 31 +-
.../trident/state/SimpleStateMapper.java | 26 +-
.../cassandra/trident/state/SimpleTuple.java | 38 +-
.../cassandra/trident/state/StateMapper.java | 25 +-
.../state/TransactionalTupleStateMapper.java | 28 +-
.../TridentAyncCQLResultSetValuesMapper.java | 28 +-
.../state/TridentResultSetValuesMapper.java | 29 +-
.../apache/storm/cassandra/WeatherSpout.java | 25 +-
.../testtools/EmbeddedCassandraResource.java | 33 +-
.../storm/cassandra/trident/MapStateTest.java | 103 +-
.../cassandra/trident/WeatherBatchSpout.java | 43 +-
external/storm-eventhubs/pom.xml | 2 +-
.../eventhubs/bolt/DefaultEventDataFormat.java | 38 +-
.../storm/eventhubs/bolt/EventHubBolt.java | 208 +-
.../eventhubs/bolt/EventHubBoltConfig.java | 147 +-
.../storm/eventhubs/bolt/IEventDataFormat.java | 3 +-
.../eventhubs/spout/BinaryEventDataScheme.java | 78 +-
.../storm/eventhubs/spout/EventDataScheme.java | 92 +-
.../storm/eventhubs/spout/EventDataWrap.java | 51 +-
.../storm/eventhubs/spout/EventHubFilter.java | 11 +-
.../eventhubs/spout/EventHubReceiverImpl.java | 237 +-
.../storm/eventhubs/spout/EventHubSpout.java | 420 ++--
.../eventhubs/spout/EventHubSpoutConfig.java | 451 ++--
.../storm/eventhubs/spout/FieldConstants.java | 13 +-
.../storm/eventhubs/spout/IEventDataScheme.java | 34 +-
.../eventhubs/spout/IEventHubReceiver.java | 11 +-
.../spout/IEventHubReceiverFactory.java | 27 +-
.../eventhubs/spout/IPartitionCoordinator.java | 5 +-
.../eventhubs/spout/IPartitionManager.java | 17 +-
.../spout/IPartitionManagerFactory.java | 33 +-
.../storm/eventhubs/spout/IStateStore.java | 9 +-
.../apache/storm/eventhubs/spout/MessageId.java | 69 +-
.../storm/eventhubs/spout/PartitionManager.java | 126 +-
.../eventhubs/spout/SimplePartitionManager.java | 200 +-
.../spout/StaticPartitionCoordinator.java | 92 +-
.../eventhubs/spout/StringEventDataScheme.java | 78 +-
.../eventhubs/spout/ZookeeperStateStore.java | 107 +-
.../storm/eventhubs/trident/Coordinator.java | 65 +-
.../trident/ITridentPartitionManager.java | 26 +-
.../ITridentPartitionManagerFactory.java | 4 +-
.../trident/OpaqueTridentEventHubEmitter.java | 69 +-
.../trident/OpaqueTridentEventHubSpout.java | 65 +-
.../storm/eventhubs/trident/Partition.java | 23 +-
.../storm/eventhubs/trident/Partitions.java | 28 +-
.../TransactionalTridentEventHubEmitter.java | 240 ++-
.../TransactionalTridentEventHubSpout.java | 64 +-
.../trident/TridentPartitionManager.java | 117 +-
.../eventhubs/samples/AtMostOnceEventCount.java | 44 +-
.../storm/eventhubs/samples/EventCount.java | 208 +-
.../storm/eventhubs/samples/EventHubLoop.java | 44 +-
.../samples/OpaqueTridentEventCount.java | 45 +-
.../samples/TransactionalTridentEventCount.java | 86 +-
.../eventhubs/samples/bolt/GlobalCountBolt.java | 95 +-
.../samples/bolt/PartialCountBolt.java | 59 +-
.../eventhubs/spout/EventHubReceiverMock.java | 97 +-
.../spout/EventHubSpoutCallerMock.java | 130 +-
.../spout/PartitionManagerCallerMock.java | 134 +-
.../spout/SpoutOutputCollectorMock.java | 76 +-
.../storm/eventhubs/spout/StateStoreMock.java | 44 +-
.../storm/eventhubs/spout/TestEventData.java | 29 +-
.../eventhubs/spout/TestEventHubSpout.java | 82 +-
.../eventhubs/spout/TestPartitionManager.java | 178 +-
.../TestTransactionalTridentEmitter.java | 123 +-
.../eventhubs/trident/TridentCollectorMock.java | 58 +-
external/storm-hbase/pom.xml | 2 +-
.../storm/hbase/bolt/AbstractHBaseBolt.java | 42 +-
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 42 +-
.../storm/hbase/bolt/HBaseLookupBolt.java | 161 +-
.../storm/hbase/bolt/mapper/HBaseMapper.java | 27 +-
.../bolt/mapper/HBaseProjectionCriteria.java | 75 +-
.../hbase/bolt/mapper/HBaseValueMapper.java | 31 +-
.../hbase/bolt/mapper/SimpleHBaseMapper.java | 55 +-
.../apache/storm/hbase/common/ColumnList.java | 166 +-
.../apache/storm/hbase/common/HBaseClient.java | 42 +-
.../org/apache/storm/hbase/common/IColumn.java | 26 +-
.../org/apache/storm/hbase/common/ICounter.java | 25 +-
.../org/apache/storm/hbase/common/Utils.java | 83 +-
.../storm/hbase/state/HBaseKeyValueState.java | 36 +-
.../hbase/state/HBaseKeyValueStateIterator.java | 42 +-
.../hbase/state/HBaseKeyValueStateProvider.java | 31 +-
.../mapper/SimpleTridentHBaseMapMapper.java | 2 +-
.../mapper/SimpleTridentHBaseMapper.java | 45 +-
.../trident/mapper/TridentHBaseMapper.java | 34 +-
.../hbase/trident/state/HBaseMapState.java | 142 +-
.../storm/hbase/trident/state/HBaseQuery.java | 24 +-
.../storm/hbase/trident/state/HBaseState.java | 125 +-
.../hbase/trident/state/HBaseStateFactory.java | 24 +-
.../storm/hbase/trident/state/HBaseUpdater.java | 24 +-
.../trident/windowing/HBaseWindowsStore.java | 65 +-
.../windowing/HBaseWindowsStoreFactory.java | 24 +-
.../storm/hbase/state/HBaseClientTestUtil.java | 38 +-
.../state/HBaseKeyValueStateIteratorTest.java | 28 +-
.../state/HBaseKeyValueStateProviderTest.java | 15 +-
.../hbase/state/HBaseKeyValueStateTest.java | 35 +-
external/storm-hdfs/pom.xml | 2 +-
.../storm/hdfs/avro/AbstractAvroSerializer.java | 33 +-
.../storm/hdfs/avro/AvroSchemaRegistry.java | 22 +-
.../org/apache/storm/hdfs/avro/AvroUtils.java | 24 +-
.../hdfs/avro/ConfluentAvroSerializer.java | 24 +-
.../storm/hdfs/avro/FixedAvroSerializer.java | 30 +-
.../storm/hdfs/avro/GenericAvroSerializer.java | 19 +-
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 79 +-
.../storm/hdfs/bolt/AvroGenericRecordBolt.java | 52 +-
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 51 +-
.../storm/hdfs/bolt/SequenceFileBolt.java | 50 +-
.../java/org/apache/storm/hdfs/bolt/Writer.java | 18 +-
.../hdfs/bolt/format/DefaultFileNameFormat.java | 32 +-
.../hdfs/bolt/format/DefaultSequenceFormat.java | 12 +-
.../hdfs/bolt/format/DelimitedRecordFormat.java | 29 +-
.../storm/hdfs/bolt/format/FileNameFormat.java | 22 +-
.../storm/hdfs/bolt/format/RecordFormat.java | 22 +-
.../storm/hdfs/bolt/format/SequenceFormat.java | 7 +-
.../hdfs/bolt/format/SimpleFileNameFormat.java | 38 +-
.../hdfs/bolt/rotation/FileRotationPolicy.java | 22 +-
.../bolt/rotation/FileSizeRotationPolicy.java | 62 +-
.../hdfs/bolt/rotation/NoRotationPolicy.java | 19 +-
.../hdfs/bolt/rotation/TimedRotationPolicy.java | 43 +-
.../storm/hdfs/bolt/sync/CountSyncPolicy.java | 21 +-
.../apache/storm/hdfs/bolt/sync/SyncPolicy.java | 22 +-
.../storm/hdfs/common/AbstractHDFSWriter.java | 22 +-
.../common/AvroGenericRecordHDFSWriter.java | 24 +-
.../apache/storm/hdfs/common/HDFSWriter.java | 26 +-
.../org/apache/storm/hdfs/common/HdfsUtils.java | 137 +-
.../storm/hdfs/common/ModifTimeComparator.java | 27 +-
.../storm/hdfs/common/NullPartitioner.java | 19 +-
.../apache/storm/hdfs/common/Partitioner.java | 24 +-
.../storm/hdfs/common/SequenceFileWriter.java | 24 +-
.../hdfs/common/rotation/MoveFileAction.java | 6 +-
.../hdfs/common/rotation/RotationAction.java | 6 +-
.../storm/hdfs/spout/AbstractFileReader.java | 70 +-
.../org/apache/storm/hdfs/spout/Configs.java | 45 +-
.../org/apache/storm/hdfs/spout/DirLock.java | 182 +-
.../org/apache/storm/hdfs/spout/FileLock.java | 563 ++---
.../org/apache/storm/hdfs/spout/FileOffset.java | 25 +-
.../org/apache/storm/hdfs/spout/FileReader.java | 49 +-
.../org/apache/storm/hdfs/spout/HdfsSpout.java | 161 +-
.../apache/storm/hdfs/spout/ParseException.java | 24 +-
.../storm/hdfs/spout/ProgressTracker.java | 86 +-
.../storm/hdfs/spout/SequenceFileReader.java | 340 ++-
.../apache/storm/hdfs/spout/TextFileReader.java | 304 ++-
.../apache/storm/hdfs/trident/HdfsState.java | 322 +--
.../storm/hdfs/trident/HdfsStateFactory.java | 12 +-
.../apache/storm/hdfs/trident/HdfsUpdater.java | 6 +-
.../trident/format/DefaultFileNameFormat.java | 29 +-
.../trident/format/DefaultSequenceFormat.java | 11 +-
.../trident/format/DelimitedRecordFormat.java | 31 +-
.../hdfs/trident/format/FileNameFormat.java | 19 +-
.../storm/hdfs/trident/format/RecordFormat.java | 22 +-
.../hdfs/trident/format/SequenceFormat.java | 7 +-
.../trident/format/SimpleFileNameFormat.java | 36 +-
.../trident/rotation/FileRotationPolicy.java | 24 +-
.../rotation/FileSizeRotationPolicy.java | 64 +-
.../hdfs/trident/rotation/NoRotationPolicy.java | 19 +-
.../trident/rotation/TimedRotationPolicy.java | 49 +-
.../hdfs/trident/sync/CountSyncPolicy.java | 21 +-
.../storm/hdfs/trident/sync/SyncPolicy.java | 22 +-
.../hdfs/avro/TestFixedAvroSerializer.java | 44 +-
.../hdfs/avro/TestGenericAvroSerializer.java | 33 +-
.../hdfs/bolt/AvroGenericRecordBoltTest.java | 129 +-
.../apache/storm/hdfs/bolt/TestHdfsBolt.java | 135 +-
.../storm/hdfs/bolt/TestSequenceFileBolt.java | 115 +-
.../apache/storm/hdfs/bolt/TestWritersMap.java | 22 +-
.../bolt/format/TestSimpleFileNameFormat.java | 39 +-
.../apache/storm/hdfs/spout/ConfigsTest.java | 46 +-
.../apache/storm/hdfs/spout/TestDirLock.java | 29 +-
.../apache/storm/hdfs/spout/TestFileLock.java | 47 +-
.../storm/hdfs/spout/TestHdfsSemantics.java | 35 +-
.../apache/storm/hdfs/spout/TestHdfsSpout.java | 201 +-
.../storm/hdfs/spout/TestProgressTracker.java | 21 +-
.../storm/hdfs/testing/MiniDFSClusterRule.java | 8 +-
.../storm/hdfs/trident/HdfsStateTest.java | 103 +-
.../format/TestSimpleFileNameFormat.java | 30 +-
external/storm-hive/pom.xml | 2 +-
.../org/apache/storm/hive/bolt/HiveBolt.java | 139 +-
.../bolt/mapper/DelimitedRecordHiveMapper.java | 60 +-
.../storm/hive/bolt/mapper/HiveMapper.java | 30 +-
.../hive/bolt/mapper/JsonRecordHiveMapper.java | 61 +-
.../apache/storm/hive/common/HiveOptions.java | 24 +-
.../org/apache/storm/hive/common/HiveUtils.java | 60 +-
.../apache/storm/hive/common/HiveWriter.java | 255 ++-
.../apache/storm/hive/trident/HiveState.java | 124 +-
.../storm/hive/trident/HiveStateFactory.java | 31 +-
.../apache/storm/hive/trident/HiveUpdater.java | 23 +-
.../apache/storm/hive/bolt/HiveSetupUtil.java | 143 +-
.../apache/storm/hive/bolt/TestHiveBolt.java | 302 ++-
.../storm/hive/common/TestHiveWriter.java | 205 +-
external/storm-jdbc/pom.xml | 2 +-
.../storm/jdbc/bolt/AbstractJdbcBolt.java | 55 +-
.../apache/storm/jdbc/bolt/JdbcInsertBolt.java | 42 +-
.../apache/storm/jdbc/bolt/JdbcLookupBolt.java | 29 +-
.../org/apache/storm/jdbc/common/Column.java | 27 +-
.../storm/jdbc/common/ConnectionProvider.java | 19 +-
.../jdbc/common/HikariCPConnectionProvider.java | 30 +-
.../apache/storm/jdbc/common/JdbcClient.java | 39 +-
.../java/org/apache/storm/jdbc/common/Util.java | 19 +-
.../storm/jdbc/mapper/JdbcLookupMapper.java | 24 +-
.../apache/storm/jdbc/mapper/JdbcMapper.java | 24 +-
.../jdbc/mapper/SimpleJdbcLookupMapper.java | 36 +-
.../storm/jdbc/mapper/SimpleJdbcMapper.java | 57 +-
.../storm/jdbc/trident/state/JdbcQuery.java | 24 +-
.../storm/jdbc/trident/state/JdbcState.java | 141 +-
.../jdbc/trident/state/JdbcStateFactory.java | 24 +-
.../storm/jdbc/trident/state/JdbcUpdater.java | 24 +-
.../storm/jdbc/bolt/JdbcInsertBoltTest.java | 28 +-
.../storm/jdbc/bolt/JdbcLookupBoltTest.java | 29 +-
.../storm/jdbc/common/JdbcClientTest.java | 63 +-
.../org/apache/storm/jdbc/common/UtilTest.java | 22 +-
external/storm-jms/pom.xml | 2 +-
.../apache/storm/jms/JmsMessageProducer.java | 21 +-
.../java/org/apache/storm/jms/JmsProvider.java | 20 +-
.../org/apache/storm/jms/JmsTupleProducer.java | 21 +-
.../java/org/apache/storm/jms/bolt/JmsBolt.java | 38 +-
.../apache/storm/jms/spout/JmsMessageID.java | 29 +-
.../org/apache/storm/jms/spout/JmsSpout.java | 216 +-
.../org/apache/storm/jms/trident/JmsBatch.java | 19 +-
.../org/apache/storm/jms/trident/JmsState.java | 103 +-
.../storm/jms/trident/JmsStateFactory.java | 22 +-
.../apache/storm/jms/trident/JmsUpdater.java | 26 +-
.../storm/jms/trident/TridentJmsSpout.java | 237 +-
.../apache/storm/jms/spout/JmsSpoutTest.java | 37 +-
.../apache/storm/jms/spout/MockJmsProvider.java | 21 +-
.../jms/spout/MockSpoutOutputCollector.java | 6 +-
.../storm/jms/spout/MockTupleProducer.java | 2 +-
external/storm-kafka-monitor/pom.xml | 2 +-
.../kafka/monitor/KafkaOffsetLagResult.java | 14 +-
.../storm/kafka/monitor/KafkaOffsetLagUtil.java | 203 +-
.../kafka/monitor/KafkaPartitionOffsetLag.java | 81 +-
.../kafka/monitor/NewKafkaSpoutOffsetQuery.java | 12 +-
.../kafka/monitor/OldKafkaSpoutOffsetQuery.java | 25 +-
external/storm-kafka/pom.xml | 2 +-
.../src/jvm/org/apache/storm/kafka/Broker.java | 51 +-
.../jvm/org/apache/storm/kafka/BrokerHosts.java | 19 +-
.../storm/kafka/ByteBufferSerializer.java | 26 +-
.../storm/kafka/DynamicBrokersReader.java | 112 +-
.../kafka/DynamicPartitionConnections.java | 55 +-
.../ExponentialBackoffMsgRetryManager.java | 75 +-
.../storm/kafka/FailedFetchException.java | 19 +-
.../storm/kafka/FailedMsgRetryManager.java | 19 +-
.../org/apache/storm/kafka/IntSerializer.java | 24 +-
.../jvm/org/apache/storm/kafka/KafkaConfig.java | 25 +-
.../jvm/org/apache/storm/kafka/KafkaError.java | 19 +-
.../jvm/org/apache/storm/kafka/KafkaSpout.java | 58 +-
.../jvm/org/apache/storm/kafka/KafkaUtils.java | 269 ++-
.../org/apache/storm/kafka/KeyValueScheme.java | 22 +-
.../kafka/KeyValueSchemeAsMultiScheme.java | 31 +-
.../storm/kafka/MessageMetadataScheme.java | 22 +-
.../MessageMetadataSchemeAsMultiScheme.java | 20 +-
.../jvm/org/apache/storm/kafka/Partition.java | 39 +-
.../storm/kafka/PartitionCoordinator.java | 19 +-
.../apache/storm/kafka/PartitionManager.java | 121 +-
.../jvm/org/apache/storm/kafka/SpoutConfig.java | 19 +-
.../apache/storm/kafka/StaticCoordinator.java | 27 +-
.../jvm/org/apache/storm/kafka/StaticHosts.java | 19 +-
.../storm/kafka/StaticPartitionConnections.java | 22 +-
.../storm/kafka/StringKeyValueScheme.java | 24 +-
.../kafka/StringMessageAndMetadataScheme.java | 27 +-
.../storm/kafka/StringMultiSchemeWithTopic.java | 28 +-
.../org/apache/storm/kafka/StringScheme.java | 38 +-
.../kafka/TopicOffsetOutOfRangeException.java | 1 +
.../org/apache/storm/kafka/ZkCoordinator.java | 50 +-
.../src/jvm/org/apache/storm/kafka/ZkHosts.java | 19 +-
.../src/jvm/org/apache/storm/kafka/ZkState.java | 65 +-
.../org/apache/storm/kafka/bolt/KafkaBolt.java | 69 +-
.../FieldNameBasedTupleToKafkaMapper.java | 21 +-
.../kafka/bolt/mapper/TupleToKafkaMapper.java | 25 +-
.../bolt/selector/DefaultTopicSelector.java | 19 +-
.../bolt/selector/FieldIndexTopicSelector.java | 19 +-
.../bolt/selector/FieldNameTopicSelector.java | 19 +-
.../kafka/bolt/selector/KafkaTopicSelector.java | 22 +-
.../apache/storm/kafka/trident/Coordinator.java | 27 +-
.../storm/kafka/trident/DefaultCoordinator.java | 19 +-
.../trident/GlobalPartitionInformation.java | 41 +-
.../storm/kafka/trident/IBatchCoordinator.java | 19 +-
.../storm/kafka/trident/IBrokerReader.java | 20 +-
.../apache/storm/kafka/trident/MaxMetric.java | 19 +-
.../kafka/trident/OpaqueTridentKafkaSpout.java | 44 +-
.../storm/kafka/trident/StaticBrokerReader.java | 23 +-
.../trident/TransactionalTridentKafkaSpout.java | 22 +-
.../storm/kafka/trident/TridentKafkaConfig.java | 19 +-
.../kafka/trident/TridentKafkaEmitter.java | 73 +-
.../storm/kafka/trident/TridentKafkaState.java | 49 +-
.../kafka/trident/TridentKafkaStateFactory.java | 34 +-
.../kafka/trident/TridentKafkaUpdater.java | 22 +-
.../storm/kafka/trident/ZkBrokerReader.java | 113 +-
.../FieldNameBasedTupleToKafkaMapper.java | 19 +-
.../mapper/TridentTupleToKafkaMapper.java | 26 +-
.../trident/selector/DefaultTopicSelector.java | 19 +-
.../trident/selector/KafkaTopicSelector.java | 22 +-
.../storm/kafka/DynamicBrokersReaderTest.java | 39 +-
.../ExponentialBackoffMsgRetryManagerTest.java | 49 +-
.../org/apache/storm/kafka/KafkaErrorTest.java | 19 +-
.../org/apache/storm/kafka/KafkaTestBroker.java | 29 +-
.../org/apache/storm/kafka/KafkaUtilsTest.java | 73 +-
.../storm/kafka/PartitionManagerTest.java | 34 +-
.../storm/kafka/StringKeyValueSchemeTest.java | 36 +-
.../apache/storm/kafka/TestStringScheme.java | 28 +-
.../test/org/apache/storm/kafka/TestUtils.java | 32 +-
.../apache/storm/kafka/TridentKafkaTest.java | 40 +-
.../apache/storm/kafka/ZkCoordinatorTest.java | 49 +-
.../apache/storm/kafka/bolt/KafkaBoltTest.java | 159 +-
external/storm-mqtt/pom.xml | 2 +-
.../java/org/apache/storm/mqtt/MqttLogger.java | 20 +-
.../java/org/apache/storm/mqtt/MqttMessage.java | 26 +-
.../apache/storm/mqtt/MqttMessageMapper.java | 22 +-
.../org/apache/storm/mqtt/MqttTupleMapper.java | 24 +-
.../org/apache/storm/mqtt/bolt/MqttBolt.java | 36 +-
.../apache/storm/mqtt/common/MqttOptions.java | 43 +-
.../apache/storm/mqtt/common/MqttPublisher.java | 28 +-
.../org/apache/storm/mqtt/common/MqttUtils.java | 34 +-
.../org/apache/storm/mqtt/common/SslUtils.java | 36 +-
.../mqtt/mappers/ByteArrayMessageMapper.java | 23 +-
.../storm/mqtt/mappers/StringMessageMapper.java | 23 +-
.../apache/storm/mqtt/spout/AckableMessage.java | 49 +-
.../org/apache/storm/mqtt/spout/MqttSpout.java | 86 +-
.../storm/mqtt/ssl/DefaultKeyStoreLoader.java | 25 +-
.../apache/storm/mqtt/ssl/KeyStoreLoader.java | 23 +-
.../storm/mqtt/trident/MqttPublishFunction.java | 33 +-
.../storm/mqtt/StormMqttIntegrationTest.java | 75 +-
external/storm-redis/pom.xml | 2 +-
.../storm/redis/bolt/AbstractRedisBolt.java | 28 +-
.../storm/redis/bolt/RedisFilterBolt.java | 22 +-
.../storm/redis/bolt/RedisLookupBolt.java | 30 +-
.../apache/storm/redis/bolt/RedisStoreBolt.java | 19 +-
.../adapter/RedisCommandsAdapterJedis.java | 7 +-
.../RedisCommandsAdapterJedisCluster.java | 7 +-
.../redis/common/commands/RedisCommands.java | 9 +-
.../redis/common/config/JedisClusterConfig.java | 23 +-
.../redis/common/config/JedisPoolConfig.java | 44 +-
.../common/container/JedisClusterContainer.java | 21 +-
.../JedisCommandsContainerBuilder.java | 26 +-
.../JedisCommandsInstanceContainer.java | 21 +-
.../redis/common/container/JedisContainer.java | 23 +-
.../common/container/RedisClusterContainer.java | 4 +-
.../RedisCommandsContainerBuilder.java | 10 +-
.../RedisCommandsInstanceContainer.java | 6 +-
.../redis/common/container/RedisContainer.java | 6 +-
.../common/mapper/RedisDataTypeDescription.java | 26 +-
.../redis/common/mapper/RedisFilterMapper.java | 19 +-
.../redis/common/mapper/RedisLookupMapper.java | 22 +-
.../storm/redis/common/mapper/RedisMapper.java | 19 +-
.../redis/common/mapper/RedisStoreMapper.java | 19 +-
.../storm/redis/common/mapper/TupleMapper.java | 22 +-
.../storm/redis/state/RedisKeyValueState.java | 62 +-
.../redis/state/RedisKeyValueStateIterator.java | 22 +-
.../redis/state/RedisKeyValueStateProvider.java | 42 +-
.../trident/state/AbstractRedisMapState.java | 37 +-
.../state/AbstractRedisStateQuerier.java | 25 +-
.../state/AbstractRedisStateUpdater.java | 30 +-
.../storm/redis/trident/state/KeyFactory.java | 21 +-
.../storm/redis/trident/state/Options.java | 24 +-
.../trident/state/RedisClusterMapState.java | 213 +-
.../redis/trident/state/RedisClusterState.java | 99 +-
.../trident/state/RedisClusterStateQuerier.java | 23 +-
.../trident/state/RedisClusterStateUpdater.java | 48 +-
.../redis/trident/state/RedisMapState.java | 264 ++-
.../storm/redis/trident/state/RedisState.java | 92 +-
.../redis/trident/state/RedisStateQuerier.java | 38 +-
.../redis/trident/state/RedisStateUpdater.java | 48 +-
.../state/RedisKeyValueStateIteratorTest.java | 69 +-
.../state/RedisKeyValueStateProviderTest.java | 34 +-
.../redis/state/RedisKeyValueStateTest.java | 186 +-
external/storm-solr/pom.xml | 2 +-
.../apache/storm/solr/bolt/SolrUpdateBolt.java | 30 +-
.../storm/solr/config/CountBasedCommit.java | 18 +-
.../storm/solr/config/SolrCommitStrategy.java | 18 +-
.../apache/storm/solr/config/SolrConfig.java | 25 +-
.../storm/solr/mapper/SolrFieldsMapper.java | 137 +-
.../storm/solr/mapper/SolrJsonMapper.java | 119 +-
.../apache/storm/solr/mapper/SolrMapper.java | 22 +-
.../storm/solr/mapper/SolrMapperException.java | 18 +-
.../org/apache/storm/solr/schema/CopyField.java | 32 +-
.../org/apache/storm/solr/schema/Field.java | 32 +-
.../org/apache/storm/solr/schema/FieldType.java | 43 +-
.../org/apache/storm/solr/schema/Schema.java | 82 +-
.../storm/solr/schema/SolrFieldTypeFinder.java | 87 +-
.../schema/builder/RestJsonSchemaBuilder.java | 21 +-
.../schema/builder/RestJsonSchemaBuilderV2.java | 49 +-
.../solr/schema/builder/SchemaBuilder.java | 20 +-
.../apache/storm/solr/trident/SolrState.java | 24 +-
.../storm/solr/trident/SolrStateFactory.java | 19 +-
.../apache/storm/solr/trident/SolrUpdater.java | 21 +-
sql/storm-sql-core/pom.xml | 2 +-
.../storm/sql/AbstractTridentProcessor.java | 73 +-
.../src/jvm/org/apache/storm/sql/StormSql.java | 50 +-
.../org/apache/storm/sql/StormSqlContext.java | 21 +-
.../jvm/org/apache/storm/sql/StormSqlImpl.java | 180 +-
.../org/apache/storm/sql/StormSqlRunner.java | 35 +-
.../sql/calcite/ParallelStreamableTable.java | 18 +-
.../apache/storm/sql/compiler/CompilerUtil.java | 262 ++-
.../sql/compiler/RexNodeToJavaCodeCompiler.java | 293 +--
.../sql/compiler/StormSqlTypeFactoryImpl.java | 23 +-
.../storm/sql/javac/CompilingClassLoader.java | 285 ++-
.../storm/sql/parser/ColumnConstraint.java | 46 +-
.../storm/sql/parser/ColumnDefinition.java | 48 +-
.../storm/sql/parser/SqlCreateFunction.java | 32 +-
.../apache/storm/sql/parser/SqlCreateTable.java | 258 ++-
.../apache/storm/sql/parser/SqlDDLKeywords.java | 21 +-
.../apache/storm/sql/parser/StormParser.java | 58 +-
.../apache/storm/sql/parser/UnparseUtil.java | 75 +-
.../sql/planner/StormRelDataTypeSystem.java | 18 +-
.../apache/storm/sql/planner/StormRelUtils.java | 24 +-
.../planner/UnsupportedOperatorsVisitor.java | 19 +-
.../storm/sql/planner/rel/StormCalcRelBase.java | 18 +-
.../sql/planner/rel/StormFilterRelBase.java | 18 +-
.../storm/sql/planner/rel/StormJoinRelBase.java | 24 +-
.../sql/planner/rel/StormProjectRelBase.java | 24 +-
.../storm/sql/planner/rel/StormRelNode.java | 18 +-
.../planner/rel/StormStreamInsertRelBase.java | 25 +-
.../sql/planner/rel/StormStreamScanRelBase.java | 18 +-
.../storm/sql/planner/trident/QueryPlanner.java | 34 +-
.../sql/planner/trident/TridentPlanCreator.java | 38 +-
.../planner/trident/TridentStormRuleSets.java | 68 +-
.../sql/planner/trident/rel/TridentCalcRel.java | 33 +-
.../planner/trident/rel/TridentFilterRel.java | 24 +-
.../trident/rel/TridentLogicalConvention.java | 97 +-
.../planner/trident/rel/TridentProjectRel.java | 29 +-
.../sql/planner/trident/rel/TridentRel.java | 19 +-
.../trident/rel/TridentStreamInsertRel.java | 34 +-
.../trident/rel/TridentStreamScanRel.java | 24 +-
.../trident/rules/TridentAggregateRule.java | 18 +-
.../planner/trident/rules/TridentCalcRule.java | 42 +-
.../trident/rules/TridentFilterRule.java | 45 +-
.../planner/trident/rules/TridentJoinRule.java | 35 +-
.../trident/rules/TridentModifyRule.java | 76 +-
.../trident/rules/TridentProjectRule.java | 46 +-
.../planner/trident/rules/TridentScanRule.java | 63 +-
.../test/org/apache/storm/sql/SqlTestUtil.java | 9 +-
.../storm/sql/StormSqlLocalClusterImpl.java | 17 +-
.../test/org/apache/storm/sql/TestStormSql.java | 177 +-
.../backends/trident/TestCompilerUtils.java | 191 +-
.../backends/trident/TestExpressions.java | 238 +-
.../backends/trident/TestPlanCompiler.java | 370 ++--
.../apache/storm/sql/parser/TestSqlParser.java | 66 +-
.../maven-shade-clojure-transformer/pom.xml | 2 +-
.../maven/shade/clojure/ClojureTransformer.java | 14 +-
storm-buildtools/storm-maven-plugins/pom.xml | 2 +-
.../storm/maven/plugin/util/CommandExec.java | 118 +-
.../plugin/versioninfo/VersionInfoMojo.java | 457 ++--
storm-client/pom.xml | 2 +-
.../src/jvm/org/apache/storm/Config.java | 1224 ++++-------
.../src/jvm/org/apache/storm/Constants.java | 21 +-
.../org/apache/storm/ICredentialsListener.java | 21 +-
.../src/jvm/org/apache/storm/ILocalCluster.java | 121 +-
.../src/jvm/org/apache/storm/ILocalDRPC.java | 21 +-
.../jvm/org/apache/storm/ISubmitterHook.java | 26 +-
.../src/jvm/org/apache/storm/LogWriter.java | 68 +-
.../jvm/org/apache/storm/StormSubmitter.java | 212 +-
.../src/jvm/org/apache/storm/StormTimer.java | 239 +-
.../src/jvm/org/apache/storm/Thrift.java | 125 +-
.../storm/annotation/InterfaceStability.java | 43 +-
.../assignments/ILocalAssignmentsBackend.java | 36 +-
.../assignments/InMemoryAssignmentBackend.java | 38 +-
.../LocalAssignmentsBackendFactory.java | 27 +-
.../storm/blobstore/AtomicOutputStream.java | 24 +-
.../org/apache/storm/blobstore/BlobStore.java | 309 ++-
.../storm/blobstore/BlobStoreAclHandler.java | 125 +-
.../apache/storm/blobstore/BlobStoreFile.java | 38 +-
.../apache/storm/blobstore/ClientBlobStore.java | 98 +-
.../storm/blobstore/InputStreamWithMeta.java | 20 +-
.../org/apache/storm/blobstore/KeyFilter.java | 19 +-
.../blobstore/LocalModeClientBlobStore.java | 21 +-
.../apache/storm/blobstore/NimbusBlobStore.java | 399 ++--
.../src/jvm/org/apache/storm/bolt/JoinBolt.java | 534 ++---
.../storm/callback/DefaultWatcherCallBack.java | 19 +-
.../apache/storm/callback/WatcherCallBack.java | 19 +-
.../storm/callback/ZKStateChangedCallback.java | 18 +-
.../storm/cluster/ClusterStateContext.java | 26 +-
.../storm/cluster/ClusterStateListener.java | 19 +-
.../org/apache/storm/cluster/ClusterUtils.java | 112 +-
.../apache/storm/cluster/ConnectionState.java | 19 +-
.../org/apache/storm/cluster/DaemonType.java | 32 +-
.../org/apache/storm/cluster/ExecutorBeat.java | 19 +-
.../org/apache/storm/cluster/IStateStorage.java | 155 +-
.../storm/cluster/IStormClusterState.java | 110 +-
.../storm/cluster/PaceMakerStateStorage.java | 44 +-
.../storm/cluster/StateStorageFactory.java | 18 +-
.../storm/cluster/StormClusterStateImpl.java | 127 +-
.../org/apache/storm/cluster/VersionedData.java | 25 +-
.../apache/storm/cluster/ZKStateStorage.java | 53 +-
.../storm/cluster/ZKStateStorageFactory.java | 3 -
.../storm/container/cgroup/CgroupCenter.java | 30 +-
.../storm/container/cgroup/CgroupCommon.java | 80 +-
.../container/cgroup/CgroupCommonOperation.java | 39 +-
.../container/cgroup/CgroupCoreFactory.java | 66 +-
.../storm/container/cgroup/CgroupOperation.java | 23 +-
.../storm/container/cgroup/CgroupUtils.java | 26 +-
.../apache/storm/container/cgroup/Device.java | 19 +-
.../storm/container/cgroup/Hierarchy.java | 19 +-
.../storm/container/cgroup/SubSystem.java | 25 +-
.../storm/container/cgroup/SubSystemType.java | 19 +-
.../storm/container/cgroup/SystemOperation.java | 39 +-
.../storm/container/cgroup/core/BlkioCore.java | 34 +-
.../storm/container/cgroup/core/CgroupCore.java | 19 +-
.../storm/container/cgroup/core/CpuCore.java | 48 +-
.../container/cgroup/core/CpuacctCore.java | 24 +-
.../storm/container/cgroup/core/CpusetCore.java | 160 +-
.../container/cgroup/core/DevicesCore.java | 117 +-
.../container/cgroup/core/FreezerCore.java | 39 +-
.../storm/container/cgroup/core/MemoryCore.java | 168 +-
.../storm/container/cgroup/core/NetClsCore.java | 24 +-
.../container/cgroup/core/NetPrioCore.java | 24 +-
.../storm/coordination/BatchBoltExecutor.java | 50 +-
.../coordination/BatchOutputCollector.java | 12 +-
.../coordination/BatchOutputCollectorImpl.java | 29 +-
.../coordination/BatchSubtopologyBuilder.java | 165 +-
.../storm/coordination/CoordinatedBolt.java | 475 ++--
.../apache/storm/coordination/IBatchBolt.java | 25 +-
.../src/jvm/org/apache/storm/daemon/Acker.java | 62 +-
.../org/apache/storm/daemon/DaemonCommon.java | 19 +-
.../org/apache/storm/daemon/GrouperFactory.java | 88 +-
.../org/apache/storm/daemon/Shutdownable.java | 19 +-
.../org/apache/storm/daemon/StormCommon.java | 253 ++-
.../src/jvm/org/apache/storm/daemon/Task.java | 70 +-
.../daemon/metrics/BuiltinBoltMetrics.java | 19 +-
.../storm/daemon/metrics/BuiltinMetrics.java | 19 +-
.../daemon/metrics/BuiltinMetricsUtil.java | 31 +-
.../daemon/metrics/BuiltinSpoutMetrics.java | 19 +-
.../daemon/metrics/ClientMetricsUtils.java | 18 +-
.../daemon/metrics/ErrorReportingMetrics.java | 18 +-
.../daemon/metrics/SpoutThrottlingMetrics.java | 18 +-
.../storm/daemon/supervisor/AdvancedFSOps.java | 258 ++-
.../supervisor/ClientSupervisorUtils.java | 70 +-
.../daemon/supervisor/ExitCodeCallback.java | 24 +-
.../storm/daemon/supervisor/IAdvancedFSOps.java | 56 +-
.../daemon/worker/BackPressureTracker.java | 10 +-
.../storm/daemon/worker/LogConfigManager.java | 17 +-
.../org/apache/storm/daemon/worker/Worker.java | 176 +-
.../apache/storm/daemon/worker/WorkerState.java | 397 ++--
.../storm/daemon/worker/WorkerTransfer.java | 6 +-
.../dependency/DependencyBlobStoreUtils.java | 3 +-
.../dependency/DependencyPropertiesParser.java | 24 +-
.../storm/dependency/DependencyUploader.java | 32 +-
.../dependency/FileNotAvailableException.java | 18 +-
.../storm/drpc/DRPCInvocationsClient.java | 50 +-
.../jvm/org/apache/storm/drpc/DRPCSpout.java | 152 +-
.../jvm/org/apache/storm/drpc/JoinResult.java | 37 +-
.../org/apache/storm/drpc/KeyedFairBolt.java | 35 +-
.../storm/drpc/LinearDRPCInputDeclarer.java | 32 +-
.../storm/drpc/LinearDRPCTopologyBuilder.java | 133 +-
.../org/apache/storm/drpc/PrepareRequest.java | 24 +-
.../org/apache/storm/drpc/ReturnResults.java | 56 +-
.../jvm/org/apache/storm/executor/Executor.java | 99 +-
.../apache/storm/executor/ExecutorShutdown.java | 33 +-
.../apache/storm/executor/ExecutorTransfer.java | 49 +-
.../apache/storm/executor/IRunningExecutor.java | 21 +-
.../apache/storm/executor/LocalExecutor.java | 25 +-
.../org/apache/storm/executor/TupleInfo.java | 26 +-
.../storm/executor/bolt/BoltExecutor.java | 50 +-
.../executor/bolt/BoltOutputCollectorImpl.java | 38 +-
.../storm/executor/error/IReportError.java | 19 +-
.../storm/executor/error/ReportError.java | 33 +-
.../storm/executor/error/ReportErrorAndDie.java | 23 +-
.../storm/executor/spout/SpoutExecutor.java | 43 +-
.../spout/SpoutOutputCollectorImpl.java | 39 +-
.../storm/grouping/CustomStreamGrouping.java | 48 +-
.../src/jvm/org/apache/storm/grouping/Load.java | 31 +-
.../grouping/LoadAwareCustomStreamGrouping.java | 21 +-
.../grouping/LoadAwareShuffleGrouping.java | 87 +-
.../org/apache/storm/grouping/LoadMapping.java | 41 +-
.../storm/grouping/PartialKeyGrouping.java | 57 +-
.../apache/storm/grouping/ShuffleGrouping.java | 30 +-
.../org/apache/storm/hooks/BaseTaskHook.java | 23 +-
.../org/apache/storm/hooks/BaseWorkerHook.java | 29 +-
.../jvm/org/apache/storm/hooks/ITaskHook.java | 34 +-
.../jvm/org/apache/storm/hooks/IWorkerHook.java | 30 +-
.../storm/hooks/SubmitterHookException.java | 1 +
.../apache/storm/hooks/info/BoltAckInfo.java | 21 +-
.../storm/hooks/info/BoltExecuteInfo.java | 21 +-
.../apache/storm/hooks/info/BoltFailInfo.java | 21 +-
.../org/apache/storm/hooks/info/EmitInfo.java | 26 +-
.../apache/storm/hooks/info/SpoutAckInfo.java | 21 +-
.../apache/storm/hooks/info/SpoutFailInfo.java | 21 +-
.../storm/lambda/LambdaBiConsumerBolt.java | 23 +-
.../apache/storm/lambda/LambdaConsumerBolt.java | 19 +-
.../org/apache/storm/lambda/LambdaSpout.java | 22 +-
.../storm/lambda/SerializableBiConsumer.java | 22 +-
.../storm/lambda/SerializableCallable.java | 18 +-
.../storm/lambda/SerializableConsumer.java | 22 +-
.../storm/lambda/SerializableSupplier.java | 22 +-
.../storm/messaging/ConnectionWithStatus.java | 59 +-
.../DeserializingConnectionCallback.java | 41 +-
.../org/apache/storm/messaging/IConnection.java | 41 +-
.../storm/messaging/IConnectionCallback.java | 20 +-
.../org/apache/storm/messaging/IContext.java | 47 +-
.../org/apache/storm/messaging/TaskMessage.java | 37 +-
.../storm/messaging/TransportFactory.java | 37 +-
.../apache/storm/messaging/local/Context.java | 138 +-
.../messaging/netty/BackPressureStatus.java | 27 +-
.../apache/storm/messaging/netty/Client.java | 158 +-
.../apache/storm/messaging/netty/Context.java | 49 +-
.../storm/messaging/netty/ControlMessage.java | 57 +-
.../messaging/netty/INettySerializable.java | 20 +-
.../storm/messaging/netty/ISaslClient.java | 23 +-
.../storm/messaging/netty/ISaslServer.java | 21 +-
.../apache/storm/messaging/netty/IServer.java | 21 +-
.../netty/KerberosSaslClientHandler.java | 31 +-
.../netty/KerberosSaslNettyClient.java | 102 +-
.../netty/KerberosSaslNettyClientState.java | 19 +-
.../netty/KerberosSaslNettyServer.java | 135 +-
.../netty/KerberosSaslNettyServerState.java | 27 +-
.../netty/KerberosSaslServerHandler.java | 40 +-
.../org/apache/storm/messaging/netty/Login.java | 137 +-
.../storm/messaging/netty/MessageBatch.java | 56 +-
.../storm/messaging/netty/MessageBuffer.java | 27 +-
.../storm/messaging/netty/MessageDecoder.java | 53 +-
.../storm/messaging/netty/MessageEncoder.java | 31 +-
.../netty/NettyRenameThreadFactory.java | 37 +-
.../netty/NettyUncaughtExceptionHandler.java | 20 +-
.../storm/messaging/netty/SaslMessageToken.java | 79 +-
.../storm/messaging/netty/SaslNettyClient.java | 68 +-
.../messaging/netty/SaslNettyClientState.java | 29 +-
.../storm/messaging/netty/SaslNettyServer.java | 73 +-
.../messaging/netty/SaslNettyServerState.java | 27 +-
.../messaging/netty/SaslStormClientHandler.java | 63 +-
.../netty/SaslStormServerAuthorizeHandler.java | 105 +-
.../messaging/netty/SaslStormServerHandler.java | 74 +-
.../apache/storm/messaging/netty/SaslUtils.java | 34 +-
.../apache/storm/messaging/netty/Server.java | 102 +-
.../messaging/netty/StormClientHandler.java | 58 +-
.../netty/StormClientPipelineFactory.java | 30 +-
.../messaging/netty/StormServerHandler.java | 66 +-
.../netty/StormServerPipelineFactory.java | 30 +-
.../apache/storm/metric/EventLoggerBolt.java | 33 +-
.../storm/metric/FileBasedEventLogger.java | 9 +-
.../org/apache/storm/metric/IEventLogger.java | 51 +-
.../storm/metric/LoggingMetricsConsumer.java | 57 +-
.../storm/metric/MetricsConsumerBolt.java | 48 +-
.../jvm/org/apache/storm/metric/SystemBolt.java | 138 +-
.../storm/metric/api/AssignableMetric.java | 19 +-
.../apache/storm/metric/api/CombinedMetric.java | 21 +-
.../apache/storm/metric/api/CountMetric.java | 21 +-
.../org/apache/storm/metric/api/ICombiner.java | 20 +-
.../org/apache/storm/metric/api/IMetric.java | 26 +-
.../storm/metric/api/IMetricsConsumer.java | 79 +-
.../storm/metric/api/IMetricsRegistrant.java | 18 +-
.../org/apache/storm/metric/api/IReducer.java | 21 +-
.../storm/metric/api/IStatefulObject.java | 19 +-
.../apache/storm/metric/api/MeanReducer.java | 14 +-
.../storm/metric/api/MultiCountMetric.java | 26 +-
.../storm/metric/api/MultiReducedMetric.java | 28 +-
.../apache/storm/metric/api/ReducedMetric.java | 19 +-
.../apache/storm/metric/api/StateMetric.java | 19 +-
.../metric/api/rpc/AssignableShellMetric.java | 19 +-
.../metric/api/rpc/CombinedShellMetric.java | 19 +-
.../storm/metric/api/rpc/CountShellMetric.java | 21 +-
.../storm/metric/api/rpc/IShellMetric.java | 19 +-
.../metric/api/rpc/ReducedShellMetric.java | 19 +-
.../apache/storm/metric/cgroup/CGroupCpu.java | 28 +-
.../storm/metric/cgroup/CGroupCpuGuarantee.java | 24 +-
.../storm/metric/cgroup/CGroupMemoryLimit.java | 24 +-
.../storm/metric/cgroup/CGroupMemoryUsage.java | 20 +-
.../storm/metric/cgroup/CGroupMetricsBase.java | 38 +-
.../storm/metric/filter/FilterByMetricName.java | 26 +-
.../storm/metric/filter/MetricsFilter.java | 22 +-
.../metric/internal/CountStatAndMetric.java | 67 +-
.../metric/internal/LatencyStatAndMetric.java | 62 +-
.../storm/metric/internal/MetricStatTimer.java | 19 +-
.../internal/MultiCountStatAndMetric.java | 43 +-
.../internal/MultiLatencyStatAndMetric.java | 44 +-
.../storm/metric/internal/RateTracker.java | 54 +-
.../storm/metric/util/DataPointExpander.java | 94 +-
.../org/apache/storm/metrics2/JcMetrics.java | 18 +-
.../org/apache/storm/metrics2/SimpleGauge.java | 18 +-
.../storm/metrics2/StormMetricRegistry.java | 38 +-
.../org/apache/storm/metrics2/TaskMetrics.java | 20 +-
.../storm/metrics2/filters/RegexFilter.java | 19 +-
.../metrics2/filters/StormMetricsFilter.java | 20 +-
.../reporters/ConsoleStormReporter.java | 18 +-
.../metrics2/reporters/CsvStormReporter.java | 66 +-
.../reporters/GangliaStormReporter.java | 80 +-
.../reporters/GraphiteStormReporter.java | 53 +-
.../metrics2/reporters/JmxStormReporter.java | 30 +-
.../reporters/ScheduledStormReporter.java | 61 +-
.../storm/metrics2/reporters/StormReporter.java | 19 +-
.../jvm/org/apache/storm/multilang/BoltMsg.java | 31 +-
.../org/apache/storm/multilang/ISerializer.java | 34 +-
.../apache/storm/multilang/JsonSerializer.java | 44 +-
.../storm/multilang/NoOutputException.java | 22 +-
.../org/apache/storm/multilang/ShellMsg.java | 106 +-
.../org/apache/storm/multilang/SpoutMsg.java | 31 +-
.../AbstractDNSToSwitchMapping.java | 52 +-
.../networktopography/DNSToSwitchMapping.java | 42 +-
.../DefaultRackDNSToSwitchMapping.java | 24 +-
.../jvm/org/apache/storm/nimbus/NimbusInfo.java | 65 +-
.../apache/storm/pacemaker/PacemakerClient.java | 125 +-
.../storm/pacemaker/PacemakerClientHandler.java | 42 +-
.../storm/pacemaker/PacemakerClientPool.java | 43 +-
.../pacemaker/PacemakerConnectionException.java | 19 +-
.../storm/pacemaker/codec/ThriftDecoder.java | 37 +-
.../storm/pacemaker/codec/ThriftEncoder.java | 67 +-
.../pacemaker/codec/ThriftNettyClientCodec.java | 53 +-
.../org/apache/storm/policy/IWaitStrategy.java | 19 +-
.../apache/storm/policy/WaitStrategyPark.java | 22 +-
.../storm/policy/WaitStrategyProgressive.java | 31 +-
.../org/apache/storm/scheduler/WorkerSlot.java | 30 +-
.../storm/security/INimbusCredentialPlugin.java | 46 +-
.../apache/storm/security/auth/AuthUtils.java | 145 +-
.../org/apache/storm/security/auth/AutoSSL.java | 134 +-
.../auth/DefaultHttpCredentialsPlugin.java | 39 +-
.../security/auth/DefaultPrincipalToLocal.java | 30 +-
.../storm/security/auth/FixedGroupsMapping.java | 25 +-
.../apache/storm/security/auth/IAuthorizer.java | 42 +-
.../storm/security/auth/IAutoCredentials.java | 37 +-
.../security/auth/ICredentialsRenewer.java | 39 +-
.../auth/IGroupMappingServiceProvider.java | 26 +-
.../security/auth/IHttpCredentialsPlugin.java | 24 +-
.../storm/security/auth/IPrincipalToLocal.java | 27 +-
.../storm/security/auth/ITransportPlugin.java | 41 +-
.../security/auth/KerberosPrincipalToLocal.java | 32 +-
.../storm/security/auth/NimbusPrincipal.java | 26 +-
.../apache/storm/security/auth/ReqContext.java | 105 +-
.../security/auth/ShellBasedGroupsMapping.java | 48 +-
.../security/auth/SimpleTransportPlugin.java | 90 +-
.../security/auth/SingleUserPrincipal.java | 18 +-
.../storm/security/auth/TBackoffConnect.java | 29 +-
.../storm/security/auth/ThriftClient.java | 60 +-
.../security/auth/ThriftConnectionType.java | 46 +-
.../storm/security/auth/ThriftServer.java | 31 +-
.../auth/authorizer/DRPCAuthorizerBase.java | 38 +-
.../authorizer/DRPCSimpleACLAuthorizer.java | 105 +-
.../auth/authorizer/DenyAuthorizer.java | 30 +-
.../authorizer/ImpersonationAuthorizer.java | 76 +-
.../auth/authorizer/NoopAuthorizer.java | 30 +-
.../auth/authorizer/SimpleACLAuthorizer.java | 129 +-
.../authorizer/SimpleWhitelistAuthorizer.java | 36 +-
.../SupervisorSimpleACLAuthorizer.java | 41 +-
.../auth/digest/DigestSaslTransportPlugin.java | 42 +-
.../auth/digest/JassPasswordProvider.java | 30 +-
.../storm/security/auth/kerberos/AutoTGT.java | 162 +-
.../auth/kerberos/AutoTGTKrb5LoginModule.java | 29 +-
.../kerberos/AutoTGTKrb5LoginModuleTest.java | 20 +-
.../auth/kerberos/ClientCallbackHandler.java | 68 +-
.../kerberos/KerberosSaslTransportPlugin.java | 223 +-
.../security/auth/kerberos/NoOpTTrasport.java | 19 +-
.../auth/kerberos/ServerCallbackHandler.java | 43 +-
.../auth/plain/PlainClientCallbackHandler.java | 23 +-
.../auth/plain/PlainSaslTransportPlugin.java | 43 +-
.../security/auth/plain/SaslPlainServer.java | 249 ++-
.../security/auth/sasl/PasswordProvider.java | 35 +-
.../security/auth/sasl/SaslTransportPlugin.java | 35 +-
.../sasl/SimpleSaslClientCallbackHandler.java | 21 +-
.../sasl/SimpleSaslServerCallbackHandler.java | 40 +-
.../auth/workertoken/WorkerTokenAuthorizer.java | 71 +-
.../WorkerTokenClientCallbackHandler.java | 50 +-
.../auth/workertoken/WorkerTokenSigner.java | 6 +-
.../serialization/BlowfishTupleSerializer.java | 66 +-
.../storm/serialization/DefaultKryoFactory.java | 65 +-
.../DefaultSerializationDelegate.java | 31 +-
.../GzipBridgeSerializationDelegate.java | 34 +-
.../GzipBridgeThriftSerializationDelegate.java | 34 +-
.../GzipSerializationDelegate.java | 31 +-
.../GzipThriftSerializationDelegate.java | 4 +-
.../storm/serialization/IKryoDecorator.java | 20 +-
.../storm/serialization/IKryoFactory.java | 37 +-
.../storm/serialization/ITupleDeserializer.java | 21 +-
.../storm/serialization/ITupleSerializer.java | 21 +-
.../serialization/KryoTupleDeserializer.java | 31 +-
.../serialization/KryoTupleSerializer.java | 47 +-
.../serialization/KryoValuesDeserializer.java | 36 +-
.../serialization/KryoValuesSerializer.java | 31 +-
.../serialization/SerializableSerializer.java | 25 +-
.../serialization/SerializationDelegate.java | 19 +-
.../serialization/SerializationFactory.java | 171 +-
.../serialization/SerializationRegister.java | 9 +-
.../ThriftSerializationDelegate.java | 26 +-
.../types/ArrayListSerializer.java | 21 +-
.../serialization/types/HashMapSerializer.java | 19 +-
.../serialization/types/HashSetSerializer.java | 21 +-
.../types/ListDelegateSerializer.java | 23 +-
.../org/apache/storm/spout/CheckPointState.java | 117 +-
.../org/apache/storm/spout/CheckpointSpout.java | 28 +-
.../storm/spout/IMultiSchemableSpout.java | 24 +-
.../org/apache/storm/spout/ISchemableSpout.java | 24 +-
.../src/jvm/org/apache/storm/spout/ISpout.java | 105 +-
.../storm/spout/ISpoutOutputCollector.java | 35 +-
.../jvm/org/apache/storm/spout/MultiScheme.java | 27 +-
.../org/apache/storm/spout/RawMultiScheme.java | 18 +-
.../jvm/org/apache/storm/spout/RawScheme.java | 8 +-
.../src/jvm/org/apache/storm/spout/Scheme.java | 21 +-
.../apache/storm/spout/SchemeAsMultiScheme.java | 49 +-
.../jvm/org/apache/storm/spout/ShellSpout.java | 80 +-
.../storm/spout/SpoutOutputCollector.java | 102 +-
.../storm/state/BaseBinaryStateIterator.java | 23 +-
.../apache/storm/state/BaseStateIterator.java | 23 +-
.../apache/storm/state/DefaultStateEncoder.java | 24 +-
.../storm/state/DefaultStateSerializer.java | 38 +-
.../jvm/org/apache/storm/state/IStateSpout.java | 24 +-
.../storm/state/IStateSpoutOutputCollector.java | 19 +-
.../apache/storm/state/ISubscribedState.java | 20 +-
.../state/ISynchronizeOutputCollector.java | 21 +-
.../storm/state/InMemoryKeyValueState.java | 71 +-
.../state/InMemoryKeyValueStateProvider.java | 22 +-
.../org/apache/storm/state/KeyValueState.java | 22 +-
.../jvm/org/apache/storm/state/Serializer.java | 22 +-
.../src/jvm/org/apache/storm/state/State.java | 29 +-
.../org/apache/storm/state/StateEncoder.java | 18 +-
.../org/apache/storm/state/StateFactory.java | 30 +-
.../org/apache/storm/state/StateProvider.java | 28 +-
.../storm/state/StateSpoutOutputCollector.java | 19 +-
.../storm/state/SynchronizeOutputCollector.java | 19 +-
.../apache/storm/stats/BoltExecutorStats.java | 34 +-
.../jvm/org/apache/storm/stats/CommonStats.java | 25 +-
.../apache/storm/stats/SpoutExecutorStats.java | 24 +-
.../jvm/org/apache/storm/stats/StatsUtil.java | 371 ++--
.../src/jvm/org/apache/storm/streams/Edge.java | 19 +-
.../org/apache/storm/streams/GroupingInfo.java | 38 +-
.../src/jvm/org/apache/storm/streams/Node.java | 72 +-
.../src/jvm/org/apache/storm/streams/Pair.java | 57 +-
.../org/apache/storm/streams/PairStream.java | 149 +-
.../org/apache/storm/streams/PartitionNode.java | 24 +-
.../org/apache/storm/streams/ProcessorBolt.java | 24 +-
.../storm/streams/ProcessorBoltDelegate.java | 60 +-
.../org/apache/storm/streams/ProcessorNode.java | 44 +-
.../apache/storm/streams/RefCountedTuple.java | 30 +-
.../jvm/org/apache/storm/streams/SinkNode.java | 22 +-
.../jvm/org/apache/storm/streams/SpoutNode.java | 21 +-
.../storm/streams/StatefulProcessorBolt.java | 30 +-
.../jvm/org/apache/storm/streams/Stream.java | 123 +-
.../org/apache/storm/streams/StreamBolt.java | 22 +-
.../org/apache/storm/streams/StreamBuilder.java | 82 +-
.../org/apache/storm/streams/StreamState.java | 22 +-
.../org/apache/storm/streams/StreamUtil.java | 24 +-
.../storm/streams/StreamsEdgeFactory.java | 22 +-
.../org/apache/storm/streams/UniqueIdGen.java | 21 +-
.../org/apache/storm/streams/WindowNode.java | 21 +-
.../storm/streams/WindowedProcessorBolt.java | 26 +-
.../storm/streams/operations/BiFunction.java | 19 +-
.../streams/operations/CombinerAggregator.java | 85 +-
.../storm/streams/operations/Consumer.java | 19 +-
.../streams/operations/FlatMapFunction.java | 19 +-
.../storm/streams/operations/Function.java | 19 +-
.../streams/operations/IdentityFunction.java | 19 +-
.../storm/streams/operations/Operation.java | 19 +-
.../streams/operations/PairFlatMapFunction.java | 19 +-
.../storm/streams/operations/PairFunction.java | 21 +-
.../streams/operations/PairValueJoiner.java | 19 +-
.../storm/streams/operations/Predicate.java | 19 +-
.../storm/streams/operations/PrintConsumer.java | 19 +-
.../storm/streams/operations/Reducer.java | 22 +-
.../storm/streams/operations/StateUpdater.java | 54 +-
.../storm/streams/operations/ValueJoiner.java | 19 +-
.../streams/operations/aggregators/Count.java | 19 +-
.../streams/operations/aggregators/LongSum.java | 19 +-
.../operations/mappers/PairValueMapper.java | 22 +-
.../operations/mappers/TupleValueMapper.java | 22 +-
.../operations/mappers/TupleValueMappers.java | 128 +-
.../streams/operations/mappers/ValueMapper.java | 22 +-
.../operations/mappers/ValuesMapper.java | 25 +-
.../processors/AggregateByKeyProcessor.java | 33 +-
.../streams/processors/AggregateProcessor.java | 27 +-
.../storm/streams/processors/BaseProcessor.java | 45 +-
.../streams/processors/BatchProcessor.java | 22 +-
.../streams/processors/BranchProcessor.java | 22 +-
.../processors/ChainedProcessorContext.java | 22 +-
.../processors/CoGroupByKeyProcessor.java | 23 +-
.../processors/EmittingProcessorContext.java | 37 +-
.../streams/processors/FilterProcessor.java | 19 +-
.../streams/processors/FlatMapProcessor.java | 19 +-
.../processors/FlatMapValuesProcessor.java | 19 +-
.../streams/processors/ForEachProcessor.java | 19 +-
.../processors/ForwardingProcessorContext.java | 25 +-
.../storm/streams/processors/JoinProcessor.java | 36 +-
.../storm/streams/processors/MapProcessor.java | 19 +-
.../streams/processors/MapValuesProcessor.java | 19 +-
.../MergeAggregateByKeyProcessor.java | 24 +-
.../processors/MergeAggregateProcessor.java | 19 +-
.../storm/streams/processors/PeekProcessor.java | 19 +-
.../storm/streams/processors/Processor.java | 29 +-
.../streams/processors/ProcessorContext.java | 28 +-
.../processors/ReduceByKeyProcessor.java | 24 +-
.../streams/processors/ReduceProcessor.java | 19 +-
.../streams/processors/StateQueryProcessor.java | 19 +-
.../streams/processors/StatefulProcessor.java | 22 +-
.../processors/UpdateStateByKeyProcessor.java | 19 +-
.../org/apache/storm/streams/tuple/Tuple10.java | 63 +-
.../org/apache/storm/streams/tuple/Tuple3.java | 35 +-
.../org/apache/storm/streams/tuple/Tuple4.java | 39 +-
.../org/apache/storm/streams/tuple/Tuple5.java | 43 +-
.../org/apache/storm/streams/tuple/Tuple6.java | 47 +-
.../org/apache/storm/streams/tuple/Tuple7.java | 51 +-
.../org/apache/storm/streams/tuple/Tuple8.java | 55 +-
.../org/apache/storm/streams/tuple/Tuple9.java | 59 +-
.../storm/streams/windowing/BaseWindow.java | 33 +-
.../storm/streams/windowing/SlidingWindows.java | 84 +-
.../streams/windowing/TumblingWindows.java | 68 +-
.../apache/storm/streams/windowing/Window.java | 32 +-
.../storm/task/GeneralTopologyContext.java | 70 +-
.../src/jvm/org/apache/storm/task/IBolt.java | 86 +-
.../org/apache/storm/task/IErrorReporter.java | 19 +-
.../org/apache/storm/task/IMetricsContext.java | 23 +-
.../org/apache/storm/task/IOutputCollector.java | 29 +-
.../org/apache/storm/task/OutputCollector.java | 164 +-
.../jvm/org/apache/storm/task/ShellBolt.java | 95 +-
.../org/apache/storm/task/TopologyContext.java | 188 +-
.../storm/task/WorkerTopologyContext.java | 110 +-
.../apache/storm/testing/AckFailDelegate.java | 20 +-
.../apache/storm/testing/AckFailMapTracker.java | 39 +-
.../org/apache/storm/testing/AckTracker.java | 31 +-
.../AlternateRackDNSToSwitchMapping.java | 79 +-
.../apache/storm/testing/BatchNumberList.java | 53 +-
.../apache/storm/testing/BatchProcessWord.java | 21 +-
.../org/apache/storm/testing/BatchRepeatA.java | 33 +-
.../org/apache/storm/testing/BoltTracker.java | 23 +-
.../apache/storm/testing/CompletableSpout.java | 21 +-
.../apache/storm/testing/CountingBatchBolt.java | 31 +-
.../storm/testing/CountingCommitBolt.java | 29 +-
.../org/apache/storm/testing/FeederSpout.java | 40 +-
.../org/apache/storm/testing/FixedTuple.java | 22 +-
.../apache/storm/testing/FixedTupleSpout.java | 108 +-
.../testing/ForwardingMetricsConsumer.java | 61 +-
.../org/apache/storm/testing/IdentityBolt.java | 23 +-
.../apache/storm/testing/IntegrationTest.java | 14 +-
.../storm/testing/KeyedCountingBatchBolt.java | 34 +-
.../testing/KeyedCountingCommitterBolt.java | 19 +-
.../storm/testing/KeyedSummingBatchBolt.java | 33 +-
.../storm/testing/MemoryTransactionalSpout.java | 185 +-
.../testing/MemoryTransactionalSpoutMeta.java | 27 +-
.../apache/storm/testing/MkClusterParam.java | 112 +-
.../org/apache/storm/testing/MkTupleParam.java | 87 +-
.../org/apache/storm/testing/MockedSources.java | 42 +-
.../jvm/org/apache/storm/testing/NGrouping.java | 37 +-
.../storm/testing/NonRichBoltTracker.java | 23 +-
.../testing/OpaqueMemoryTransactionalSpout.java | 163 +-
.../apache/storm/testing/PrepareBatchBolt.java | 30 +-
.../storm/testing/PythonShellMetricsBolt.java | 30 +-
.../storm/testing/PythonShellMetricsSpout.java | 34 +-
.../testing/SingleUserSimpleTransport.java | 45 +-
.../org/apache/storm/testing/SpoutTracker.java | 100 +-
.../storm/testing/TestAggregatesCounter.java | 34 +-
.../org/apache/storm/testing/TestConfBolt.java | 31 +-
.../apache/storm/testing/TestEventLogSpout.java | 114 +-
.../storm/testing/TestEventOrderCheckBolt.java | 35 +-
.../apache/storm/testing/TestGlobalCount.java | 30 +-
.../apache/storm/testing/TestKryoDecorator.java | 24 +-
.../apache/storm/testing/TestPlannerBolt.java | 31 +-
.../apache/storm/testing/TestPlannerSpout.java | 56 +-
.../org/apache/storm/testing/TestSerObject.java | 27 +-
.../storm/testing/TestWordBytesCounter.java | 21 +-
.../apache/storm/testing/TestWordCounter.java | 42 +-
.../org/apache/storm/testing/TestWordSpout.java | 48 +-
.../jvm/org/apache/storm/testing/TmpPath.java | 44 +-
.../apache/storm/testing/TupleCaptureBolt.java | 37 +-
.../topology/BaseConfigurationDeclarer.java | 40 +-
.../topology/BaseStatefulBoltExecutor.java | 57 +-
.../storm/topology/BasicBoltExecutor.java | 31 +-
.../storm/topology/BasicOutputCollector.java | 33 +-
.../org/apache/storm/topology/BoltDeclarer.java | 22 +-
.../topology/CheckpointTupleForwarder.java | 49 +-
.../ComponentConfigurationDeclarer.java | 26 +-
.../storm/topology/ConfigurableTopology.java | 94 +-
.../apache/storm/topology/FailedException.java | 23 +-
.../org/apache/storm/topology/IBasicBolt.java | 25 +-
.../storm/topology/IBasicOutputCollector.java | 26 +-
.../org/apache/storm/topology/IComponent.java | 28 +-
.../org/apache/storm/topology/IRichBolt.java | 24 +-
.../org/apache/storm/topology/IRichSpout.java | 24 +-
.../apache/storm/topology/IRichStateSpout.java | 19 +-
.../apache/storm/topology/IStatefulBolt.java | 30 +-
.../storm/topology/IStatefulComponent.java | 34 +-
.../storm/topology/IStatefulWindowedBolt.java | 21 +-
.../apache/storm/topology/IWindowedBolt.java | 30 +-
.../apache/storm/topology/InputDeclarer.java | 77 +-
.../storm/topology/OutputFieldsDeclarer.java | 23 +-
.../storm/topology/OutputFieldsGetter.java | 25 +-
.../PersistentWindowedBoltExecutor.java | 38 +-
.../storm/topology/ReportedFailedException.java | 23 +-
.../apache/storm/topology/ResourceDeclarer.java | 37 +-
.../storm/topology/SharedOffHeapWithinNode.java | 22 +-
.../topology/SharedOffHeapWithinWorker.java | 22 +-
.../org/apache/storm/topology/SharedOnHeap.java | 22 +-
.../apache/storm/topology/SpoutDeclarer.java | 21 +-
.../storm/topology/StatefulBoltExecutor.java | 37 +-
.../topology/StatefulWindowedBoltExecutor.java | 80 +-
.../apache/storm/topology/TopologyBuilder.java | 379 ++--
.../topology/TupleFieldTimestampExtractor.java | 31 +-
.../storm/topology/WindowedBoltExecutor.java | 75 +-
.../storm/topology/base/BaseBasicBolt.java | 23 +-
.../storm/topology/base/BaseBatchBolt.java | 21 +-
.../storm/topology/base/BaseComponent.java | 23 +-
...BaseOpaquePartitionedTransactionalSpout.java | 21 +-
.../base/BasePartitionedTransactionalSpout.java | 19 +-
.../storm/topology/base/BaseRichBolt.java | 21 +-
.../storm/topology/base/BaseRichSpout.java | 19 +-
.../storm/topology/base/BaseStatefulBolt.java | 22 +-
.../topology/base/BaseStatefulWindowedBolt.java | 31 +-
.../base/BaseTickTupleAwareRichBolt.java | 8 +-
.../topology/base/BaseTransactionalBolt.java | 21 +-
.../topology/base/BaseTransactionalSpout.java | 19 +-
.../storm/topology/base/BaseWindowedBolt.java | 325 ++-
.../apache/storm/transactional/ICommitter.java | 25 +-
.../ICommitterTransactionalSpout.java | 29 +-
.../transactional/ITransactionalSpout.java | 95 +-
.../storm/transactional/TransactionAttempt.java | 37 +-
.../TransactionalSpoutBatchExecutor.java | 39 +-
.../TransactionalSpoutCoordinator.java | 139 +-
.../TransactionalTopologyBuilder.java | 265 ++-
.../IOpaquePartitionedTransactionalSpout.java | 48 +-
.../IPartitionedTransactionalSpout.java | 63 +-
...uePartitionedTransactionalSpoutExecutor.java | 118 +-
.../PartitionedTransactionalSpoutExecutor.java | 114 +-
.../state/RotatingTransactionalState.java | 102 +-
.../state/TestTransactionalState.java | 31 +-
.../transactional/state/TransactionalState.java | 108 +-
.../apache/storm/trident/JoinOutFieldsMode.java | 29 +-
.../jvm/org/apache/storm/trident/JoinType.java | 21 +-
.../jvm/org/apache/storm/trident/Stream.java | 387 ++--
.../org/apache/storm/trident/TridentState.java | 19 +-
.../apache/storm/trident/TridentTopology.java | 1463 ++++++-------
.../trident/drpc/ReturnResultsReducer.java | 78 +-
.../fluent/ChainedAggregatorDeclarer.java | 119 +-
.../fluent/ChainedFullAggregatorDeclarer.java | 26 +-
.../ChainedPartitionAggregatorDeclarer.java | 28 +-
.../trident/fluent/GlobalAggregationScheme.java | 20 +-
.../storm/trident/fluent/GroupedStream.java | 79 +-
.../trident/fluent/IAggregatableStream.java | 25 +-
.../fluent/IChainedAggregatorDeclarer.java | 19 +-
.../storm/trident/fluent/UniqueIdGen.java | 21 +-
.../storm/trident/graph/GraphGrouper.java | 89 +-
.../org/apache/storm/trident/graph/Group.java | 99 +-
.../storm/trident/operation/Aggregator.java | 21 +-
.../storm/trident/operation/Assembly.java | 25 +-
.../storm/trident/operation/BaseAggregator.java | 21 +-
.../storm/trident/operation/BaseFilter.java | 21 +-
.../storm/trident/operation/BaseFunction.java | 21 +-
.../trident/operation/BaseMultiReducer.java | 21 +-
.../storm/trident/operation/BaseOperation.java | 29 +-
.../trident/operation/CombinerAggregator.java | 21 +-
.../storm/trident/operation/Consumer.java | 25 +-
.../operation/DefaultResourceDeclarer.java | 34 +-
.../storm/trident/operation/EachOperation.java | 21 +-
.../apache/storm/trident/operation/Filter.java | 31 +-
.../trident/operation/FlatMapFunction.java | 23 +-
.../storm/trident/operation/Function.java | 86 +-
.../trident/operation/GroupedMultiReducer.java | 23 +-
.../trident/operation/ITridentResource.java | 26 +-
.../storm/trident/operation/MapFunction.java | 22 +-
.../storm/trident/operation/MultiReducer.java | 23 +-
.../storm/trident/operation/Operation.java | 35 +-
.../OperationAwareFlatMapFunction.java | 19 +-
.../operation/OperationAwareMapFunction.java | 19 +-
.../trident/operation/ReducerAggregator.java | 20 +-
.../trident/operation/TridentCollector.java | 33 +-
.../operation/TridentMultiReducerContext.java | 29 +-
.../operation/TridentOperationContext.java | 37 +-
.../operation/builtin/ComparisonAggregator.java | 41 +-
.../storm/trident/operation/builtin/Count.java | 21 +-
.../storm/trident/operation/builtin/Debug.java | 27 +-
.../storm/trident/operation/builtin/Equals.java | 27 +-
.../trident/operation/builtin/FilterNull.java | 26 +-
.../storm/trident/operation/builtin/FirstN.java | 65 +-
.../storm/trident/operation/builtin/MapGet.java | 27 +-
.../storm/trident/operation/builtin/Max.java | 25 +-
.../operation/builtin/MaxWithComparator.java | 31 +-
.../storm/trident/operation/builtin/Min.java | 25 +-
.../operation/builtin/MinWithComparator.java | 29 +-
.../storm/trident/operation/builtin/Negate.java | 47 +-
.../trident/operation/builtin/SnapshotGet.java | 21 +-
.../storm/trident/operation/builtin/Sum.java | 40 +-
.../operation/builtin/TupleCollectionGet.java | 23 +-
.../operation/impl/CaptureCollector.java | 25 +-
.../operation/impl/ChainedAggregatorImpl.java | 72 +-
.../trident/operation/impl/ChainedResult.java | 35 +-
.../operation/impl/CombinerAggStateUpdater.java | 31 +-
.../impl/CombinerAggregatorCombineImpl.java | 41 +-
.../impl/CombinerAggregatorInitImpl.java | 27 +-
.../operation/impl/ConsumerExecutor.java | 19 +-
.../trident/operation/impl/FilterExecutor.java | 25 +-
.../operation/impl/FlatMapFunctionExecutor.java | 22 +-
.../operation/impl/GlobalBatchToPartition.java | 21 +-
.../trident/operation/impl/GroupCollector.java | 27 +-
.../operation/impl/GroupedAggregator.java | 37 +-
.../impl/GroupedMultiReducerExecutor.java | 39 +-
.../operation/impl/IdentityMultiReducer.java | 21 +-
.../impl/IndexHashBatchToPartition.java | 21 +-
.../storm/trident/operation/impl/JoinState.java | 24 +-
.../operation/impl/JoinerMultiReducer.java | 59 +-
.../operation/impl/MapFunctionExecutor.java | 23 +-
...PreservingFieldsOrderJoinerMultiReducer.java | 38 +-
.../operation/impl/ReducerAggStateUpdater.java | 29 +-
.../operation/impl/ReducerAggregatorImpl.java | 39 +-
.../storm/trident/operation/impl/Result.java | 21 +-
.../operation/impl/SingleEmitAggregator.java | 68 +-
.../trident/operation/impl/TrueFilter.java | 21 +-
.../storm/trident/partition/GlobalGrouping.java | 27 +-
.../trident/partition/IdentityGrouping.java | 35 +-
.../trident/partition/IndexHashGrouping.java | 45 +-
.../storm/trident/planner/BridgeReceiver.java | 23 +-
.../org/apache/storm/trident/planner/Node.java | 26 +-
.../storm/trident/planner/NodeStateInfo.java | 21 +-
.../storm/trident/planner/PartitionNode.java | 27 +-
.../storm/trident/planner/ProcessorContext.java | 21 +-
.../storm/trident/planner/ProcessorNode.java | 23 +-
.../apache/storm/trident/planner/SpoutNode.java | 31 +-
.../storm/trident/planner/SubtopologyBolt.java | 110 +-
.../storm/trident/planner/TridentProcessor.java | 30 +-
.../storm/trident/planner/TupleReceiver.java | 20 +-
.../planner/processor/AggregateProcessor.java | 31 +-
.../planner/processor/AppendCollector.java | 31 +-
.../planner/processor/EachProcessor.java | 33 +-
.../planner/processor/FreshCollector.java | 33 +-
.../trident/planner/processor/MapProcessor.java | 30 +-
.../processor/MultiReducerProcessor.java | 36 +-
.../processor/PartitionPersistProcessor.java | 45 +-
.../planner/processor/ProjectedProcessor.java | 33 +-
.../planner/processor/StateQueryProcessor.java | 52 +-
.../planner/processor/TridentContext.java | 45 +-
.../storm/trident/spout/BatchSpoutExecutor.java | 85 +-
.../apache/storm/trident/spout/IBatchID.java | 20 +-
.../apache/storm/trident/spout/IBatchSpout.java | 29 +-
.../trident/spout/ICommitterTridentSpout.java | 29 +-
.../spout/IOpaquePartitionedTridentSpout.java | 95 +-
.../trident/spout/IPartitionedTridentSpout.java | 75 +-
.../storm/trident/spout/ISpoutPartition.java | 19 +-
.../storm/trident/spout/ITridentDataSource.java | 1 +
.../storm/trident/spout/ITridentSpout.java | 131 +-
.../OpaquePartitionedTridentSpoutExecutor.java | 164 +-
.../spout/PartitionedTridentSpoutExecutor.java | 138 +-
.../trident/spout/RichSpoutBatchExecutor.java | 196 +-
.../storm/trident/spout/RichSpoutBatchId.java | 29 +-
.../spout/RichSpoutBatchIdSerializer.java | 21 +-
.../trident/spout/RichSpoutBatchTriggerer.java | 85 +-
.../trident/spout/TridentSpoutCoordinator.java | 37 +-
.../trident/spout/TridentSpoutExecutor.java | 65 +-
.../storm/trident/state/BaseQueryFunction.java | 21 +-
.../storm/trident/state/BaseStateUpdater.java | 21 +-
.../trident/state/CombinerValueUpdater.java | 28 +-
.../storm/trident/state/ITupleCollection.java | 19 +-
.../state/JSONNonTransactionalSerializer.java | 21 +-
.../trident/state/JSONOpaqueSerializer.java | 21 +-
.../state/JSONTransactionalSerializer.java | 21 +-
.../apache/storm/trident/state/OpaqueValue.java | 39 +-
.../storm/trident/state/QueryFunction.java | 20 +-
.../storm/trident/state/ReadOnlyState.java | 21 +-
.../trident/state/ReducerValueUpdater.java | 27 +-
.../apache/storm/trident/state/Serializer.java | 20 +-
.../org/apache/storm/trident/state/State.java | 41 +-
.../storm/trident/state/StateFactory.java | 21 +-
.../apache/storm/trident/state/StateSpec.java | 21 +-
.../apache/storm/trident/state/StateType.java | 19 +-
.../storm/trident/state/StateUpdater.java | 19 +-
.../storm/trident/state/TransactionalValue.java | 27 +-
.../storm/trident/state/ValueUpdater.java | 19 +-
.../trident/state/map/CachedBatchReadsMap.java | 56 +-
.../storm/trident/state/map/CachedMap.java | 30 +-
.../storm/trident/state/map/IBackingMap.java | 24 +-
.../state/map/MapCombinerAggStateUpdater.java | 42 +-
.../state/map/MapReducerAggStateUpdater.java | 35 +-
.../storm/trident/state/map/MapState.java | 20 +-
.../state/map/MicroBatchIBackingMap.java | 51 +-
.../trident/state/map/NonTransactionalMap.java | 38 +-
.../storm/trident/state/map/OpaqueMap.java | 62 +-
.../trident/state/map/ReadOnlyMapState.java | 19 +-
.../trident/state/map/RemovableMapState.java | 19 +-
.../trident/state/map/SnapshottableMap.java | 21 +-
.../trident/state/map/TransactionalMap.java | 52 +-
.../state/snapshot/ReadOnlySnapshottable.java | 21 +-
.../trident/state/snapshot/Snapshottable.java | 20 +-
.../trident/testing/CountAsAggregator.java | 31 +-
.../storm/trident/testing/FeederBatchSpout.java | 181 +-
.../testing/FeederCommitterBatchSpout.java | 86 +-
.../storm/trident/testing/FixedBatchSpout.java | 50 +-
.../apache/storm/trident/testing/IFeeder.java | 21 +-
.../trident/testing/LRUMemoryMapState.java | 46 +-
.../storm/trident/testing/MemoryBackingMap.java | 26 +-
.../storm/trident/testing/MemoryMapState.java | 58 +-
.../org/apache/storm/trident/testing/Split.java | 27 +-
.../storm/trident/testing/StringLength.java | 23 +-
.../storm/trident/testing/TrueFilter.java | 21 +-
.../storm/trident/testing/TuplifyArgs.java | 25 +-
.../storm/trident/topology/BatchInfo.java | 21 +-
.../trident/topology/ITridentBatchBolt.java | 27 +-
.../topology/MasterBatchCoordinator.java | 211 +-
.../trident/topology/TransactionAttempt.java | 39 +-
.../trident/topology/TridentBoltExecutor.java | 489 ++---
.../topology/TridentTopologyBuilder.java | 360 ++--
.../state/RotatingTransactionalState.java | 107 +-
.../topology/state/TestTransactionalState.java | 31 +-
.../topology/state/TransactionalState.java | 115 +-
.../apache/storm/trident/tuple/ComboList.java | 85 +-
.../apache/storm/trident/tuple/ConsList.java | 24 +-
.../storm/trident/tuple/TridentTuple.java | 24 +-
.../storm/trident/tuple/TridentTupleView.java | 359 ++-
.../storm/trident/tuple/ValuePointer.java | 52 +-
.../storm/trident/util/ErrorEdgeFactory.java | 21 +-
.../apache/storm/trident/util/IndexedEdge.java | 23 +-
.../org/apache/storm/trident/util/LRUMap.java | 21 +-
.../apache/storm/trident/util/TridentUtils.java | 80 +-
.../windowing/AbstractTridentWindowManager.java | 119 +-
.../windowing/ITridentWindowManager.java | 33 +-
.../windowing/InMemoryTridentWindowManager.java | 22 +-
.../trident/windowing/InMemoryWindowsStore.java | 56 +-
.../windowing/InMemoryWindowsStoreFactory.java | 34 +-
.../StoreBasedTridentWindowManager.java | 59 +-
.../trident/windowing/TridentBatchTuple.java | 20 +-
.../trident/windowing/WindowKryoSerializer.java | 28 +-
.../windowing/WindowTridentProcessor.java | 114 +-
.../storm/trident/windowing/WindowsState.java | 25 +-
.../trident/windowing/WindowsStateFactory.java | 24 +-
.../trident/windowing/WindowsStateUpdater.java | 33 +-
.../storm/trident/windowing/WindowsStore.java | 26 +-
.../trident/windowing/WindowsStoreFactory.java | 30 +-
.../windowing/config/BaseWindowConfig.java | 23 +-
.../windowing/config/SlidingCountWindow.java | 33 +-
.../windowing/config/SlidingDurationWindow.java | 33 +-
.../windowing/config/TumblingCountWindow.java | 31 +-
.../config/TumblingDurationWindow.java | 28 +-
.../trident/windowing/config/WindowConfig.java | 25 +-
.../windowing/strategy/BaseWindowStrategy.java | 20 +-
.../strategy/SlidingCountWindowStrategy.java | 24 +-
.../strategy/SlidingDurationWindowStrategy.java | 25 +-
.../strategy/TumblingCountWindowStrategy.java | 27 +-
.../TumblingDurationWindowStrategy.java | 25 +-
.../windowing/strategy/WindowStrategy.java | 24 +-
.../org/apache/storm/tuple/AddressedTuple.java | 33 +-
.../src/jvm/org/apache/storm/tuple/Fields.java | 72 +-
.../src/jvm/org/apache/storm/tuple/ITuple.java | 135 +-
.../jvm/org/apache/storm/tuple/MessageId.java | 55 +-
.../src/jvm/org/apache/storm/tuple/Tuple.java | 45 +-
.../jvm/org/apache/storm/tuple/TupleImpl.java | 62 +-
.../src/jvm/org/apache/storm/tuple/Values.java | 30 +-
.../jvm/org/apache/storm/utils/BatchHelper.java | 21 +-
.../storm/utils/BufferFileInputStream.java | 25 +-
.../apache/storm/utils/CRC32OutputStream.java | 25 +-
.../jvm/org/apache/storm/utils/ConfigUtils.java | 53 +-
.../org/apache/storm/utils/CuratorUtils.java | 38 +-
.../jvm/org/apache/storm/utils/DRPCClient.java | 113 +-
.../storm/utils/DefaultShellLogHandler.java | 44 +-
.../utils/DisallowedStrategyException.java | 6 +-
.../storm/utils/ExtendedThreadPoolExecutor.java | 66 +-
.../jvm/org/apache/storm/utils/IPredicate.java | 26 +-
.../org/apache/storm/utils/InprocMessaging.java | 42 +-
.../src/jvm/org/apache/storm/utils/JCQueue.java | 454 ++--
.../storm/utils/KeyedRoundRobinQueue.java | 29 +-
.../org/apache/storm/utils/ListDelegate.java | 37 +-
.../jvm/org/apache/storm/utils/LocalState.java | 66 +-
.../jvm/org/apache/storm/utils/MutableInt.java | 29 +-
.../jvm/org/apache/storm/utils/MutableLong.java | 29 +-
.../org/apache/storm/utils/MutableObject.java | 33 +-
.../org/apache/storm/utils/NimbusClient.java | 129 +-
.../utils/NimbusLeaderNotFoundException.java | 8 +-
.../org/apache/storm/utils/ObjectReader.java | 10 +-
.../org/apache/storm/utils/ReflectionUtils.java | 17 +-
.../storm/utils/RegisteredGlobalState.java | 50 +-
.../jvm/org/apache/storm/utils/RotatingMap.java | 75 +-
.../org/apache/storm/utils/ServiceRegistry.java | 37 +-
.../storm/utils/ShellBoltMessageQueue.java | 42 +-
.../apache/storm/utils/ShellCommandRunner.java | 39 +-
.../storm/utils/ShellCommandRunnerImpl.java | 2 +-
.../org/apache/storm/utils/ShellLogHandler.java | 40 +-
.../org/apache/storm/utils/ShellProcess.java | 59 +-
.../jvm/org/apache/storm/utils/ShellUtils.java | 365 ++--
.../org/apache/storm/utils/SimpleVersion.java | 44 +-
.../StormBoundedExponentialBackoffRetry.java | 48 +-
.../apache/storm/utils/SupervisorClient.java | 68 +-
.../apache/storm/utils/ThriftTopologyUtils.java | 23 +-
.../src/jvm/org/apache/storm/utils/Time.java | 161 +-
.../org/apache/storm/utils/TimeCacheMap.java | 62 +-
.../org/apache/storm/utils/TransferDrainer.java | 19 +-
.../jvm/org/apache/storm/utils/TupleUtils.java | 60 +-
.../src/jvm/org/apache/storm/utils/Utils.java | 566 ++---
.../jvm/org/apache/storm/utils/VersionInfo.java | 205 +-
.../org/apache/storm/utils/VersionedStore.java | 83 +-
.../storm/utils/WindowedTimeThrottler.java | 31 +-
.../org/apache/storm/utils/WritableUtils.java | 662 +++---
.../apache/storm/utils/ZookeeperAuthInfo.java | 29 +-
.../storm/utils/ZookeeperServerCnxnFactory.java | 123 +-
.../storm/validation/ConfigValidation.java | 610 +++---
.../validation/ConfigValidationAnnotations.java | 82 +-
.../storm/validation/ConfigValidationUtils.java | 120 +-
.../org/apache/storm/validation/NotConf.java | 21 +-
.../org/apache/storm/validation/Validated.java | 19 +-
.../storm/windowing/CountEvictionPolicy.java | 36 +-
.../storm/windowing/CountTriggerPolicy.java | 30 +-
.../storm/windowing/DefaultEvictionContext.java | 21 +-
.../jvm/org/apache/storm/windowing/Event.java | 27 +-
.../org/apache/storm/windowing/EventImpl.java | 25 +-
.../apache/storm/windowing/EvictionContext.java | 27 +-
.../apache/storm/windowing/EvictionPolicy.java | 90 +-
.../storm/windowing/StatefulWindowManager.java | 67 +-
.../storm/windowing/TimeEvictionPolicy.java | 46 +-
.../storm/windowing/TimeTriggerPolicy.java | 49 +-
.../storm/windowing/TimestampExtractor.java | 22 +-
.../apache/storm/windowing/TriggerHandler.java | 22 +-
.../apache/storm/windowing/TriggerPolicy.java | 25 +-
.../org/apache/storm/windowing/TupleWindow.java | 19 +-
.../apache/storm/windowing/TupleWindowImpl.java | 46 +-
.../storm/windowing/TupleWindowIterImpl.java | 18 +-
.../apache/storm/windowing/WaterMarkEvent.java | 22 +-
.../windowing/WaterMarkEventGenerator.java | 47 +-
.../windowing/WatermarkCountEvictionPolicy.java | 31 +-
.../windowing/WatermarkCountTriggerPolicy.java | 33 +-
.../windowing/WatermarkTimeEvictionPolicy.java | 44 +-
.../windowing/WatermarkTimeTriggerPolicy.java | 47 +-
.../jvm/org/apache/storm/windowing/Window.java | 45 +-
.../windowing/WindowLifecycleListener.java | 39 +-
.../apache/storm/windowing/WindowManager.java | 108 +-
.../persistence/SimpleWindowPartitionCache.java | 50 +-
.../persistence/WindowPartitionCache.java | 21 +-
.../windowing/persistence/WindowState.java | 39 +-
.../apache/storm/zookeeper/ClientZookeeper.java | 48 +-
.../apache/storm/zookeeper/ZkEventTypes.java | 22 +-
.../apache/storm/zookeeper/ZkKeeperStates.java | 22 +-
.../storm/PaceMakerStateStorageFactoryTest.java | 92 +-
.../org/apache/storm/TestConfigValidate.java | 225 +-
.../LocalAssignmentsBackendTest.java | 17 +-
.../storm/blobstore/ClientBlobStoreTest.java | 290 ++-
.../jvm/org/apache/storm/bolt/TestJoinBolt.java | 299 ++-
.../apache/storm/cluster/DaemonTypeTest.java | 21 +-
.../cluster/StormClusterStateImplTest.java | 36 +-
.../daemon/worker/LogConfigManagerTest.java | 112 +-
.../apache/storm/daemon/worker/WorkerTest.java | 22 +-
.../DependencyPropertiesParserTest.java | 26 +-
.../dependency/DependencyUploaderTest.java | 29 +-
.../storm/executor/error/ReportErrorTest.java | 38 +-
.../grouping/LoadAwareShuffleGroupingTest.java | 87 +-
.../storm/grouping/ShuffleGroupingTest.java | 32 +-
.../BalancedTargetSelectorTest.java | 32 +-
.../PartialKeyGroupingTest.java | 22 +-
.../RandomTwoTaskAssignmentCreatorTest.java | 19 +-
.../DeserializingConnectionCallbackTest.java | 33 +-
.../metric/filter/FilterByMetricNameTest.java | 29 +-
.../metric/internal/CountStatAndMetricTest.java | 47 +-
.../internal/LatencyStatAndMetricTest.java | 39 +-
.../storm/metric/internal/RateTrackerTest.java | 30 +-
.../metric/util/DataPointExpanderTest.java | 172 +-
.../storm/security/auth/AuthUtilsTest.java | 93 +-
.../storm/security/auth/AuthUtilsTestMock.java | 36 +-
.../apache/storm/security/auth/AutoSSLTest.java | 81 +-
.../auth/DefaultHttpCredentialsPluginTest.java | 29 +-
.../storm/security/auth/ReqContextTest.java | 28 +-
.../security/auth/SaslTransportPluginTest.java | 19 +-
.../auth/ShellBasedGroupsMappingTest.java | 40 +-
.../storm/security/auth/ThriftClientTest.java | 29 +-
.../authorizer/DRPCSimpleACLAuthorizerTest.java | 132 +-
.../authorizer/SimpleACLAuthorizerTest.java | 505 +++--
.../BlowfishTupleSerializerTest.java | 51 +-
.../GzipBridgeSerializationDelegateTest.java | 31 +-
...ipBridgeThriftSerializationDelegateTest.java | 28 +-
.../serialization/SerializationFactoryTest.java | 23 +-
.../ThriftBridgeSerializationDelegateTest.java | 22 +-
.../apache/storm/spout/CheckpointSpoutTest.java | 38 +-
.../state/BaseBinaryStateIteratorTest.java | 255 ++-
.../storm/state/DefaultStateSerializerTest.java | 8 +-
.../storm/state/InMemoryKeyValueStateTest.java | 50 +-
.../apache/storm/streams/ProcessorBoltTest.java | 44 +-
.../streams/StatefulProcessorBoltTest.java | 30 +-
.../apache/storm/streams/StreamBuilderTest.java | 133 +-
.../streams/WindowedProcessorBoltTest.java | 26 +-
.../processors/CoGroupByKeyProcessorTest.java | 49 +-
.../streams/processors/JoinProcessorTest.java | 51 +-
.../PersistentWindowedBoltExecutorTest.java | 68 +-
.../SimpleWindowPartitionCacheTest.java | 31 +-
.../topology/StatefulBoltExecutorTest.java | 36 +-
.../StatefulWindowedBoltExecutorTest.java | 30 +-
.../storm/topology/TopologyBuilderTest.java | 41 +-
.../topology/WindowedBoltExecutorTest.java | 84 +-
.../storm/trident/TestTridentTopology.java | 39 +-
.../storm/trident/TridentWindowingTest.java | 65 +-
.../jvm/org/apache/storm/tuple/FieldsTest.java | 35 +-
.../org/apache/storm/utils/ConfigUtilsTest.java | 24 +-
.../apache/storm/utils/CuratorUtilsTest.java | 15 +-
.../storm/utils/DefaultShellLogHandlerTest.java | 27 +-
.../storm/utils/JCQueueBackpressureTest.java | 69 +-
.../jvm/org/apache/storm/utils/JCQueueTest.java | 74 +-
.../apache/storm/utils/MockTupleHelpers.java | 42 +-
.../storm/utils/ShellBoltMessageQueueTest.java | 26 +-
.../org/apache/storm/utils/ShellUtilsTest.java | 50 +-
...StormBoundedExponentialBackoffRetryTest.java | 43 +-
.../storm/utils/ThriftTopologyUtilsTest.java | 46 +-
.../utils/ThrowableNestedCauseMatcher.java | 9 +-
.../jvm/org/apache/storm/utils/TimeTest.java | 36 +-
.../jvm/org/apache/storm/utils/UtilsTest.java | 90 +-
.../utils/ZookeeperServerCnxnFactoryTest.java | 19 +-
.../windowing/WaterMarkEventGeneratorTest.java | 33 +-
.../storm/windowing/WindowManagerTest.java | 154 +-
.../windowing/persistence/WindowStateTest.java | 47 +-
storm-core/pom.xml | 2 +-
.../jvm/org/apache/storm/command/Activate.java | 32 +-
.../org/apache/storm/command/AdminCommands.java | 39 +-
.../apache/storm/command/BasicDrpcClient.java | 30 +-
.../jvm/org/apache/storm/command/Blobstore.java | 85 +-
.../src/jvm/org/apache/storm/command/CLI.java | 303 ++-
.../org/apache/storm/command/ConfigValue.java | 26 +-
.../org/apache/storm/command/Deactivate.java | 32 +-
.../org/apache/storm/command/DevZookeeper.java | 20 +-
.../jvm/org/apache/storm/command/GetErrors.java | 28 +-
.../org/apache/storm/command/HealthCheck.java | 20 +-
.../org/apache/storm/command/Heartbeats.java | 23 +-
.../org/apache/storm/command/KillTopology.java | 39 +-
.../org/apache/storm/command/KillWorkers.java | 24 +-
.../apache/storm/command/ListTopologies.java | 55 +-
.../jvm/org/apache/storm/command/Monitor.java | 31 +-
.../jvm/org/apache/storm/command/Rebalance.java | 39 +-
.../org/apache/storm/command/SetLogLevel.java | 32 +-
.../apache/storm/command/UploadCredentials.java | 40 +-
.../org/apache/storm/planner/CompoundSpout.java | 21 +-
.../org/apache/storm/planner/CompoundTask.java | 21 +-
.../org/apache/storm/planner/TaskBundle.java | 25 +-
.../apache/storm/testing/MockLeaderElector.java | 23 +-
.../staticmocking/MockedClientZookeeper.java | 1 +
.../testing/staticmocking/MockedCluster.java | 20 +-
.../staticmocking/MockedSupervisorUtils.java | 18 +-
.../testing/staticmocking/MockedZookeeper.java | 18 +-
.../apache/storm/ui/FilterConfiguration.java | 19 +-
.../jvm/org/apache/storm/ui/IConfigurator.java | 19 +-
.../storm/ui/InvalidRequestException.java | 1 +
.../src/jvm/org/apache/storm/ui/UIHelpers.java | 70 +-
.../apache/storm/utils/ClojureTimerTask.java | 25 +-
.../src/jvm/org/apache/storm/utils/Monitor.java | 185 +-
.../apache/storm/utils/TopologySpoutLag.java | 75 +-
.../test/jvm/org/apache/storm/MockAutoCred.java | 25 +-
.../org/apache/storm/command/RebalanceTest.java | 21 +-
.../apache/storm/command/SetLogLevelTest.java | 21 +-
.../jvm/org/apache/storm/command/TestCLI.java | 48 +-
.../apache/storm/metric/FakeMetricConsumer.java | 41 +-
.../nimbus/InMemoryTopologyActionNotifier.java | 27 +-
.../storm/serialization/SerializationTest.java | 10 +-
.../org/apache/storm/stats/TestStatsUtil.java | 64 +-
.../staticmocking/ConfigUtilsInstaller.java | 3 +-
.../staticmocking/ReflectionUtilsInstaller.java | 3 +-
.../ServerConfigUtilsInstaller.java | 3 +-
.../utils/staticmocking/UtilsInstaller.java | 3 +-
.../storm/utils/staticmocking/package-info.java | 66 +-
storm-server/pom.xml | 2 +-
.../java/org/apache/storm/DaemonConfig.java | 454 ++--
.../ILocalClusterTrackedTopologyAware.java | 25 +-
.../java/org/apache/storm/LocalCluster.java | 809 ++++---
.../main/java/org/apache/storm/LocalDRPC.java | 12 +-
.../java/org/apache/storm/ProcessSimulator.java | 34 +-
.../src/main/java/org/apache/storm/Testing.java | 330 ++-
.../storm/blobstore/BlobKeySequenceInfo.java | 35 +-
.../apache/storm/blobstore/BlobStoreUtils.java | 35 +-
.../storm/blobstore/BlobSynchronizer.java | 49 +-
.../storm/blobstore/FileBlobStoreImpl.java | 154 +-
.../storm/blobstore/KeySequenceNumber.java | 38 +-
.../storm/blobstore/LocalFsBlobStore.java | 84 +-
.../storm/blobstore/LocalFsBlobStoreFile.java | 36 +-
.../container/ResourceIsolationInterface.java | 18 +-
.../storm/container/cgroup/CgroupManager.java | 85 +-
.../daemon/drpc/BlockingOutstandingRequest.java | 6 +-
.../java/org/apache/storm/daemon/drpc/DRPC.java | 87 +-
.../apache/storm/daemon/drpc/DRPCThrift.java | 3 +-
.../storm/daemon/drpc/OutstandingRequest.java | 4 +-
.../storm/daemon/drpc/RequestFactory.java | 3 +-
.../storm/daemon/metrics/MetricsUtils.java | 1 -
.../reporters/ConsolePreparableReporter.java | 18 +-
.../reporters/CsvPreparableReporter.java | 18 +-
.../reporters/JmxPreparableReporter.java | 18 +-
.../metrics/reporters/PreparableReporter.java | 28 +-
.../org/apache/storm/daemon/nimbus/Nimbus.java | 2040 +++++++++---------
.../apache/storm/daemon/nimbus/TopoCache.java | 46 +-
.../storm/daemon/nimbus/TopologyActions.java | 23 +-
.../storm/daemon/nimbus/TopologyResources.java | 107 +-
.../daemon/nimbus/TopologyStateTransition.java | 23 +-
.../storm/daemon/supervisor/BasicContainer.java | 675 +++---
.../supervisor/BasicContainerLauncher.java | 32 +-
.../storm/daemon/supervisor/Container.java | 305 +--
.../daemon/supervisor/ContainerLauncher.java | 48 +-
.../supervisor/ContainerRecoveryException.java | 19 +-
.../DefaultUncaughtExceptionHandler.java | 20 +-
.../supervisor/EventManagerPushCallback.java | 19 +-
.../storm/daemon/supervisor/Killable.java | 27 +-
.../storm/daemon/supervisor/LocalContainer.java | 26 +-
.../supervisor/LocalContainerLauncher.java | 22 +-
.../daemon/supervisor/OnlyLatestExecutor.java | 9 +-
.../daemon/supervisor/ReadClusterState.java | 109 +-
.../daemon/supervisor/RunAsUserContainer.java | 25 +-
.../supervisor/RunAsUserContainerLauncher.java | 34 +-
.../apache/storm/daemon/supervisor/Slot.java | 863 ++++----
.../daemon/supervisor/StandaloneSupervisor.java | 30 +-
.../storm/daemon/supervisor/Supervisor.java | 184 +-
.../daemon/supervisor/SupervisorUtils.java | 59 +-
.../apache/storm/daemon/supervisor/UniFunc.java | 19 +-
.../timer/ReportWorkerHeartbeats.java | 19 +-
.../supervisor/timer/SupervisorHealthCheck.java | 1 -
.../supervisor/timer/SupervisorHeartbeat.java | 24 +-
.../timer/SynchronizeAssignments.java | 43 +-
.../org/apache/storm/event/EventManager.java | 19 +-
.../org/apache/storm/event/EventManagerImp.java | 30 +-
.../apache/storm/healthcheck/HealthChecker.java | 19 +-
.../apache/storm/localizer/AsyncLocalizer.java | 278 +--
.../storm/localizer/BlobChangingCallback.java | 14 +-
.../org/apache/storm/localizer/GoodToGo.java | 46 +-
.../apache/storm/localizer/LocalResource.java | 18 +-
.../storm/localizer/LocalizedResource.java | 93 +-
.../LocalizedResourceRetentionSet.java | 28 +-
.../storm/localizer/LocallyCachedBlob.java | 115 +-
.../localizer/LocallyCachedTopologyBlob.java | 164 +-
.../storm/localizer/PortAndAssignment.java | 18 +-
.../storm/logging/ThriftAccessLogger.java | 20 +-
.../logging/filters/AccessLoggingFilter.java | 21 +-
.../metric/ClusterMetricsConsumerExecutor.java | 32 +-
.../metric/LoggingClusterMetricsConsumer.java | 38 +-
.../storm/metric/StormMetricsRegistry.java | 29 +-
.../org/apache/storm/metric/api/DataPoint.java | 19 +-
.../metric/api/IClusterMetricsConsumer.java | 32 +-
.../org/apache/storm/metricstore/AggLevel.java | 21 +-
.../apache/storm/metricstore/FilterOptions.java | 79 +-
.../org/apache/storm/metricstore/Metric.java | 82 +-
.../storm/metricstore/MetricException.java | 19 +-
.../apache/storm/metricstore/MetricStore.java | 21 +-
.../storm/metricstore/MetricStoreConfig.java | 23 +-
.../metricstore/NimbusMetricProcessor.java | 21 +-
.../metricstore/WorkerMetricsProcessor.java | 19 +-
.../storm/metricstore/rocksdb/KeyType.java | 32 +-
.../metricstore/rocksdb/MetricsCleaner.java | 19 +-
.../rocksdb/ReadOnlyStringMetadataCache.java | 25 +-
.../storm/metricstore/rocksdb/RocksDbKey.java | 114 +-
.../rocksdb/RocksDbMetricsWriter.java | 35 +-
.../storm/metricstore/rocksdb/RocksDbStore.java | 50 +-
.../storm/metricstore/rocksdb/RocksDbValue.java | 25 +-
.../metricstore/rocksdb/StringMetadata.java | 19 +-
.../rocksdb/StringMetadataCache.java | 57 +-
.../rocksdb/WritableStringMetadataCache.java | 22 +-
.../nimbus/AssignmentDistributionService.java | 130 +-
.../storm/nimbus/DefaultTopologyValidator.java | 22 +-
.../org/apache/storm/nimbus/ILeaderElector.java | 21 +-
.../nimbus/ITopologyActionNotifierPlugin.java | 19 +-
.../apache/storm/nimbus/ITopologyValidator.java | 23 +-
.../IWorkerHeartbeatsRecoveryStrategy.java | 18 +-
.../storm/nimbus/LeaderListenerCallback.java | 58 +-
.../nimbus/NimbusHeartbeatsPressureTest.java | 124 +-
.../storm/nimbus/StrictTopologyValidator.java | 22 +-
...TimeOutWorkerHeartbeatsRecoveryStrategy.java | 20 +-
...WorkerHeartbeatsRecoveryStrategyFactory.java | 24 +-
.../storm/pacemaker/IServerMessageHandler.java | 19 +-
.../org/apache/storm/pacemaker/Pacemaker.java | 123 +-
.../apache/storm/pacemaker/PacemakerServer.java | 109 +-
.../pacemaker/codec/ThriftNettyServerCodec.java | 67 +-
.../org/apache/storm/scheduler/Cluster.java | 219 +-
.../org/apache/storm/scheduler/Component.java | 22 +-
.../storm/scheduler/DefaultScheduler.java | 9 +-
.../apache/storm/scheduler/EvenScheduler.java | 4 +-
.../apache/storm/scheduler/ExecutorDetails.java | 6 +-
.../org/apache/storm/scheduler/INimbus.java | 22 +-
.../org/apache/storm/scheduler/IScheduler.java | 22 +-
.../storm/scheduler/ISchedulingState.java | 118 +-
.../org/apache/storm/scheduler/ISupervisor.java | 21 +-
.../storm/scheduler/IsolationScheduler.java | 51 +-
.../storm/scheduler/SchedulerAssignment.java | 20 +-
.../scheduler/SchedulerAssignmentImpl.java | 35 +-
.../storm/scheduler/SingleTopologyCluster.java | 2 +-
.../storm/scheduler/SupervisorDetails.java | 51 +-
.../storm/scheduler/SupervisorResources.java | 4 +-
.../org/apache/storm/scheduler/Topologies.java | 30 +-
.../apache/storm/scheduler/TopologyDetails.java | 94 +-
.../scheduler/blacklist/BlacklistScheduler.java | 44 +-
.../blacklist/reporters/IReporter.java | 18 +-
.../blacklist/reporters/LogReporter.java | 21 +-
.../strategies/DefaultBlacklistStrategy.java | 29 +-
.../strategies/IBlacklistStrategy.java | 19 +-
.../scheduler/multitenant/DefaultPool.java | 363 ++--
.../storm/scheduler/multitenant/FreePool.java | 182 +-
.../scheduler/multitenant/IsolatedPool.java | 617 +++---
.../multitenant/MultitenantScheduler.java | 186 +-
.../storm/scheduler/multitenant/Node.java | 561 +++--
.../storm/scheduler/multitenant/NodePool.java | 494 +++--
.../storm/scheduler/resource/RAS_Node.java | 101 +-
.../storm/scheduler/resource/RAS_Nodes.java | 8 +-
.../resource/ResourceAwareScheduler.java | 90 +-
.../storm/scheduler/resource/ResourceUtils.java | 34 +-
.../scheduler/resource/SchedulingResult.java | 20 +-
.../scheduler/resource/SchedulingStatus.java | 20 +-
.../apache/storm/scheduler/resource/User.java | 10 +-
.../normalization/NormalizedResourceOffer.java | 9 +-
.../NormalizedResourceRequest.java | 54 +-
.../normalization/NormalizedResources.java | 66 +-
.../normalization/ResourceNameNormalizer.java | 10 +-
.../DefaultSchedulingPriorityStrategy.java | 90 +-
.../FIFOSchedulingPriorityStrategy.java | 28 +-
.../priority/ISchedulingPriorityStrategy.java | 19 +-
.../scheduling/BaseResourceAwareStrategy.java | 231 +-
.../scheduling/ConstraintSolverStrategy.java | 709 +++---
.../DefaultResourceAwareStrategy.java | 93 +-
.../GenericResourceAwareStrategy.java | 129 +-
.../strategies/scheduling/IStrategy.java | 23 +-
.../utils/ArtifactoryConfigLoader.java | 115 +-
.../utils/ArtifactoryConfigLoaderFactory.java | 18 +-
.../utils/ConfigLoaderFactoryService.java | 18 +-
.../storm/scheduler/utils/FileConfigLoader.java | 19 +-
.../utils/FileConfigLoaderFactory.java | 18 +-
.../storm/scheduler/utils/IConfigLoader.java | 20 +-
.../scheduler/utils/IConfigLoaderFactory.java | 18 +-
.../auth/workertoken/WorkerTokenManager.java | 64 +-
.../storm/testing/CompleteTopologyParam.java | 146 +-
.../storm/testing/InProcessZookeeper.java | 27 +-
.../java/org/apache/storm/testing/TestJob.java | 31 +-
.../apache/storm/testing/TrackedTopology.java | 164 +-
.../apache/storm/utils/BufferInputStream.java | 18 +-
.../java/org/apache/storm/utils/LruMap.java | 20 +-
.../apache/storm/utils/ServerConfigUtils.java | 16 +-
.../org/apache/storm/utils/ServerUtils.java | 201 +-
.../storm/utils/StormCommonInstaller.java | 20 +-
.../apache/storm/zookeeper/AclEnforcement.java | 42 +-
.../storm/zookeeper/LeaderElectorImp.java | 30 +-
.../org/apache/storm/zookeeper/Zookeeper.java | 54 +-
.../java/org/apache/storm/DaemonConfigTest.java | 42 +-
.../java/org/apache/storm/LocalStateTest.java | 38 +-
.../java/org/apache/storm/MessagingTest.java | 38 +-
.../java/org/apache/storm/PacemakerTest.java | 25 +-
.../test/java/org/apache/storm/TestCgroups.java | 51 +-
.../apache/storm/TestDaemonConfigValidate.java | 24 +-
.../java/org/apache/storm/TestRebalance.java | 47 +-
.../test/java/org/apache/storm/TestingTest.java | 50 +-
.../java/org/apache/storm/TickTupleTest.java | 41 +-
.../apache/storm/blobstore/BlobStoreTest.java | 798 ++++---
.../storm/blobstore/BlobStoreUtilsTest.java | 34 +-
.../storm/blobstore/BlobSynchronizerTest.java | 201 +-
.../blobstore/MockZookeeperClientBuilder.java | 30 +-
.../org/apache/storm/daemon/drpc/DRPCTest.java | 60 +-
.../apache/storm/daemon/nimbus/NimbusTest.java | 3 +-
.../daemon/supervisor/BasicContainerTest.java | 776 +++----
.../storm/daemon/supervisor/ContainerTest.java | 231 +-
.../storm/daemon/supervisor/SlotTest.java | 259 +--
.../storm/localizer/AsyncLocalizerTest.java | 342 ++-
.../LocalizedResourceRetentionSetTest.java | 31 +-
.../ClusterMetricsConsumerExecutorTest.java | 73 +-
.../apache/storm/nimbus/LocalNimbusTest.java | 36 +-
.../org/apache/storm/scheduler/ClusterTest.java | 24 +-
.../resource/TestResourceAwareScheduler.java | 99 +-
.../storm/scheduler/resource/TestUser.java | 40 +-
.../TestUtilsForResourceAwareScheduler.java | 88 +-
.../utils/ArtifactoryConfigLoaderTest.java | 114 +-
.../scheduler/utils/FileConfigLoaderTest.java | 29 +-
.../apache/storm/security/auth/AuthTest.java | 356 +--
1790 files changed, 50515 insertions(+), 58810 deletions(-)
----------------------------------------------------------------------
[33/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-cassandra
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/impl/DefaultClient.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/impl/DefaultClient.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/impl/DefaultClient.java
index 6eb8e16..d8495cc 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/impl/DefaultClient.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/client/impl/DefaultClient.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.client.impl;
import com.datastax.driver.core.Cluster;
@@ -24,13 +18,12 @@ import com.datastax.driver.core.Metadata;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.exceptions.NoHostAvailableException;
import com.google.common.base.Preconditions;
-import org.apache.storm.cassandra.client.SimpleClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.Closeable;
import java.io.Serializable;
import java.util.Set;
+import org.apache.storm.cassandra.client.SimpleClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Simple class to wrap cassandra {@link com.datastax.driver.core.Cluster} instance.
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/BaseBeanFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/BaseBeanFactory.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/BaseBeanFactory.java
index b528223..0c1964e 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/BaseBeanFactory.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/BaseBeanFactory.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.context;
import java.util.Map;
@@ -28,6 +22,7 @@ public abstract class BaseBeanFactory<T> implements BeanFactory<T> {
protected WorkerCtx context;
protected volatile T instance;
+
/**
* {@inheritDoc}
*/
@@ -41,13 +36,15 @@ public abstract class BaseBeanFactory<T> implements BeanFactory<T> {
*/
@Override
public synchronized T get(Map<String, Object> topoConf) {
- if( instance != null) return instance;
+ if (instance != null) return instance;
return instance = make(topoConf);
}
+
/**
* Return a new instance of T.
*/
protected abstract T make(final Map<String, Object> topoConf);
+
/**
* {@inheritDoc}
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/BeanFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/BeanFactory.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/BeanFactory.java
index b573b51..fb712a5 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/BeanFactory.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/BeanFactory.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.context;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/WorkerCtx.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/WorkerCtx.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/WorkerCtx.java
index 15668d1..1a457f9 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/WorkerCtx.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/context/WorkerCtx.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.context;
import java.io.Serializable;
@@ -54,7 +48,7 @@ public class WorkerCtx implements Serializable {
*/
protected <T> BeanFactory<T> getBeanfactory(Class<T> clazz) {
BeanFactory<T> factory = (BeanFactory<T>) this.componentCtx.get(clazz);
- if( factory == null) throw new RuntimeException("Cannot resolve bean factory for class : " + clazz.getCanonicalName());
+ if (factory == null) throw new RuntimeException("Cannot resolve bean factory for class : " + clazz.getCanonicalName());
factory.setStormContext(this);
return factory;
}
@@ -64,7 +58,7 @@ public class WorkerCtx implements Serializable {
* @return
*/
public <T, K, V> T getWorkerBean(Class<T> clazz, Map<K, V> topoConf) {
- return getWorkerBean(clazz, topoConf,false);
+ return getWorkerBean(clazz, topoConf, false);
}
/**
@@ -77,13 +71,13 @@ public class WorkerCtx implements Serializable {
* @return a instance of type {@link T}.
*/
public <T, K, V> T getWorkerBean(Class<T> clazz, Map<K, V> topoConf, boolean force) {
- if( force ) workerCtx.remove(clazz);
- BeanFactory<T> factory = (BeanFactory<T>) this.workerCtx.get(clazz);
- if( factory == null) {
+ if (force) workerCtx.remove(clazz);
+ BeanFactory<T> factory = (BeanFactory<T>) this.workerCtx.get(clazz);
+ if (factory == null) {
BeanFactory<T> instance = getBeanfactory(clazz).newInstance();
workerCtx.putIfAbsent(clazz, instance);
factory = (BeanFactory<T>) this.workerCtx.get(clazz);
}
- return factory.get((Map<String, Object>)topoConf);
+ return factory.get((Map<String, Object>) topoConf);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncExecutor.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncExecutor.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncExecutor.java
index 63b81fe..53e6ee6 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncExecutor.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncExecutor.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.executor;
import com.datastax.driver.core.ResultSet;
@@ -27,10 +21,6 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.storm.topology.FailedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
@@ -39,6 +29,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.storm.topology.FailedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Service to asynchronously executes cassandra statements.
@@ -69,7 +62,7 @@ public class AsyncExecutor<T> implements Serializable {
* @param executorService The executor service responsible to execute handler.
*/
private AsyncExecutor(Session session, ExecutorService executorService, AsyncResultHandler<T> handler) {
- this.session = session;
+ this.session = session;
this.executorService = executorService;
this.handler = handler;
}
@@ -86,11 +79,11 @@ public class AsyncExecutor<T> implements Serializable {
List<SettableFuture<T>> settableFutures = new ArrayList<>(statements.size());
- for(Statement s : statements)
+ for (Statement s : statements)
settableFutures.add(execAsync(s, input, AsyncResultHandler.NO_OP_HANDLER));
ListenableFuture<List<T>> allAsList = Futures.allAsList(settableFutures);
- Futures.addCallback(allAsList, new FutureCallback<List<T>>(){
+ Futures.addCallback(allAsList, new FutureCallback<List<T>>() {
@Override
public void onSuccess(List<T> inputs) {
handler.success(input);
@@ -147,7 +140,8 @@ public class AsyncExecutor<T> implements Serializable {
* Asynchronously executes the specified select statements. Results will be passed to the {@link AsyncResultSetHandler}
* once each query has succeed or failed.
*/
- public SettableFuture<List<T>> execAsync(final List<Statement> statements, final List<T> inputs, Semaphore throttle, final AsyncResultSetHandler<T> handler) {
+ public SettableFuture<List<T>> execAsync(final List<Statement> statements, final List<T> inputs, Semaphore throttle,
+ final AsyncResultSetHandler<T> handler) {
final SettableFuture<List<T>> settableFuture = SettableFuture.create();
if (inputs.size() == 0) {
@@ -184,11 +178,10 @@ public class AsyncExecutor<T> implements Serializable {
handler.failure(throwable, input);
} catch (Throwable throwable2) {
asyncContext.exception(throwable2);
- }
- finally {
+ } finally {
asyncContext
- .exception(throwable)
- .release();
+ .exception(throwable)
+ .release();
pending.decrementAndGet();
LOG.error(String.format("Failed to execute statement '%s' ", statement), throwable);
}
@@ -196,7 +189,7 @@ public class AsyncExecutor<T> implements Serializable {
}, executorService);
} catch (Throwable throwable) {
asyncContext.exception(throwable)
- .release();
+ .release();
pending.decrementAndGet();
break;
}
@@ -207,12 +200,26 @@ public class AsyncExecutor<T> implements Serializable {
return settableFuture;
}
+ /**
+ * Returns the number of currently executed tasks which are not yet completed.
+ */
+ public int getPendingTasksSize() {
+ return this.pending.intValue();
+ }
+
+ public void shutdown() {
+ if (!executorService.isShutdown()) {
+ LOG.info("shutting down async handler executor");
+ this.executorService.shutdownNow();
+ }
+ }
+
private static class AsyncContext<T> {
private final List<T> inputs;
private final SettableFuture<List<T>> future;
- private final AtomicInteger latch;
- private final List<Throwable> exceptions;
- private final Semaphore throttle;
+ private final AtomicInteger latch;
+ private final List<Throwable> exceptions;
+ private final Semaphore throttle;
public AsyncContext(List<T> inputs, Semaphore throttle, SettableFuture<List<T>> settableFuture) {
this.inputs = inputs;
@@ -238,8 +245,7 @@ public class AsyncExecutor<T> implements Serializable {
if (remaining == 0) {
if (exceptions.size() == 0) {
future.set(inputs);
- }
- else {
+ } else {
future.setException(new MultiFailedException(exceptions));
}
@@ -254,20 +260,6 @@ public class AsyncExecutor<T> implements Serializable {
}
}
- /**
- * Returns the number of currently executed tasks which are not yet completed.
- */
- public int getPendingTasksSize() {
- return this.pending.intValue();
- }
-
- public void shutdown( ) {
- if( ! executorService.isShutdown() ) {
- LOG.info("shutting down async handler executor");
- this.executorService.shutdownNow();
- }
- }
-
public static class MultiFailedException extends FailedException {
private final List<Throwable> exceptions;
@@ -280,12 +272,12 @@ public class AsyncExecutor<T> implements Serializable {
int top5 = Math.min(exceptions.size(), 5);
StringBuilder sb = new StringBuilder();
sb.append("First ")
- .append(top5)
- .append(" exceptions: ")
- .append(System.lineSeparator());
+ .append(top5)
+ .append(" exceptions: ")
+ .append(System.lineSeparator());
for (int i = 0; i < top5; i++) {
sb.append(exceptions.get(i).getMessage())
- .append(System.lineSeparator());
+ .append(System.lineSeparator());
}
return sb.toString();
}
@@ -295,13 +287,13 @@ public class AsyncExecutor<T> implements Serializable {
StringBuilder sb = new StringBuilder();
sb.append(getMessage())
- .append(System.lineSeparator())
- .append("Multiple exceptions encountered: ")
- .append(System.lineSeparator());
+ .append(System.lineSeparator())
+ .append("Multiple exceptions encountered: ")
+ .append(System.lineSeparator());
for (Throwable exception : exceptions) {
sb.append(exception.toString())
- .append(System.lineSeparator());
+ .append(System.lineSeparator());
}
return super.toString();
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncExecutorProvider.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncExecutorProvider.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncExecutorProvider.java
index f4b7277..62df53c 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncExecutorProvider.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncExecutorProvider.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.executor;
import com.datastax.driver.core.Session;
@@ -32,7 +26,7 @@ public class AsyncExecutorProvider {
*/
public static <T> AsyncExecutor getLocal(Session session, AsyncResultHandler<T> handler) {
AsyncExecutor<T> executor = localAsyncExecutor.<T>get();
- if( executor == null ) {
+ if (executor == null) {
localAsyncExecutor.set(executor = new AsyncExecutor<>(session, handler));
}
return executor;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncResultHandler.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncResultHandler.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncResultHandler.java
index f827d45..2f8766e 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncResultHandler.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncResultHandler.java
@@ -1,26 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.cassandra.executor;
-import org.apache.storm.task.OutputCollector;
+package org.apache.storm.cassandra.executor;
import java.io.Serializable;
+import org.apache.storm.task.OutputCollector;
/**
* Default handler for batch asynchronous execution.
@@ -57,8 +50,8 @@ public interface AsyncResultHandler<T> extends Serializable {
*
* @param inputs The input tuple proceed.
*/
- void success(T inputs) ;
+ void success(T inputs);
void flush(OutputCollector collector);
-
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncResultSetHandler.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncResultSetHandler.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncResultSetHandler.java
index 8ccb400..7a2159d 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncResultSetHandler.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/AsyncResultSetHandler.java
@@ -1,25 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.executor;
import com.datastax.driver.core.ResultSet;
-
import java.io.Serializable;
/**
@@ -53,6 +46,6 @@ public interface AsyncResultSetHandler<T> extends Serializable {
*
* @param inputs The input tuple proceed.
*/
- void success(T inputs, ResultSet resultSet) ;
+ void success(T inputs, ResultSet resultSet);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/ExecutionResultCollector.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/ExecutionResultCollector.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/ExecutionResultCollector.java
index 882aeb4..44721ad 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/ExecutionResultCollector.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/ExecutionResultCollector.java
@@ -1,29 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.executor;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Tuple;
import com.google.common.collect.Lists;
-import org.apache.storm.cassandra.ExecutionResultHandler;
-
import java.util.List;
+import org.apache.storm.cassandra.ExecutionResultHandler;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.tuple.Tuple;
/**
* This class is responsible to collect input tuples proceed.
@@ -41,7 +34,7 @@ public interface ExecutionResultCollector {
* @param input the input tuple.
*/
public SucceedCollector(Tuple input) {
- this(Lists.newArrayList(input));
+ this(Lists.newArrayList(input));
}
/**
@@ -58,8 +51,8 @@ public interface ExecutionResultCollector {
*/
@Override
public void handle(OutputCollector collector, ExecutionResultHandler handler) {
- for(Tuple t : inputs) handler.onQuerySuccess(collector, t);
- for(Tuple t : inputs) collector.ack(t);
+ for (Tuple t : inputs) handler.onQuerySuccess(collector, t);
+ for (Tuple t : inputs) collector.ack(t);
}
}
@@ -88,7 +81,8 @@ public interface ExecutionResultCollector {
}
/**
- * Calls {@link ExecutionResultHandler#onThrowable(Throwable, org.apache.storm.task.OutputCollector, org.apache.storm.tuple.Tuple)} .
+ * Calls
+ * {@link ExecutionResultHandler#onThrowable(Throwable, org.apache.storm.task.OutputCollector, org.apache.storm.tuple.Tuple)} .
*/
@Override
public void handle(OutputCollector collector, ExecutionResultHandler handler) {
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/impl/BatchAsyncResultHandler.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/impl/BatchAsyncResultHandler.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/impl/BatchAsyncResultHandler.java
index f7a8fcc..99a27ef 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/impl/BatchAsyncResultHandler.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/impl/BatchAsyncResultHandler.java
@@ -1,31 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.executor.impl;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Tuple;
+import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
import org.apache.storm.cassandra.ExecutionResultHandler;
import org.apache.storm.cassandra.executor.AsyncResultHandler;
import org.apache.storm.cassandra.executor.ExecutionResultCollector;
-
-import java.util.List;
-import java.util.concurrent.ConcurrentLinkedQueue;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.tuple.Tuple;
public class BatchAsyncResultHandler implements AsyncResultHandler<List<Tuple>> {
@@ -51,6 +44,7 @@ public class BatchAsyncResultHandler implements AsyncResultHandler<List<Tuple>>
public void failure(Throwable t, List<Tuple> input) {
completed.offer(new ExecutionResultCollector.FailedCollector(input, t));
}
+
/**
* This method is responsible for acknowledging specified inputs.
*
@@ -66,7 +60,7 @@ public class BatchAsyncResultHandler implements AsyncResultHandler<List<Tuple>>
@Override
public void flush(final OutputCollector collector) {
ExecutionResultCollector poll;
- while( (poll = completed.poll()) != null ) {
+ while ((poll = completed.poll()) != null) {
poll.handle(collector, handler);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/impl/SingleAsyncResultHandler.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/impl/SingleAsyncResultHandler.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/impl/SingleAsyncResultHandler.java
index ac79543..d7cc19b 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/impl/SingleAsyncResultHandler.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/executor/impl/SingleAsyncResultHandler.java
@@ -1,30 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.executor.impl;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.tuple.Tuple;
+import java.util.concurrent.ConcurrentLinkedQueue;
import org.apache.storm.cassandra.ExecutionResultHandler;
import org.apache.storm.cassandra.executor.AsyncResultHandler;
import org.apache.storm.cassandra.executor.ExecutionResultCollector;
-
-import java.util.concurrent.ConcurrentLinkedQueue;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.tuple.Tuple;
public class SingleAsyncResultHandler implements AsyncResultHandler<Tuple> {
@@ -50,6 +43,7 @@ public class SingleAsyncResultHandler implements AsyncResultHandler<Tuple> {
public void failure(Throwable t, Tuple input) {
completed.offer(new ExecutionResultCollector.FailedCollector(input, t));
}
+
/**
* This method is responsible for acknowledging specified inputs.
*
@@ -65,7 +59,7 @@ public class SingleAsyncResultHandler implements AsyncResultHandler<Tuple> {
@Override
public void flush(final OutputCollector collector) {
ExecutionResultCollector poll;
- while( (poll = completed.poll()) != null ) {
+ while ((poll = completed.poll()) != null) {
poll.handle(collector, handler);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/AyncCQLResultSetValuesMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/AyncCQLResultSetValuesMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/AyncCQLResultSetValuesMapper.java
index 9b92b99..f57a99c 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/AyncCQLResultSetValuesMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/AyncCQLResultSetValuesMapper.java
@@ -1,30 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
-import org.apache.storm.tuple.ITuple;
-import org.apache.storm.tuple.Values;
-
import java.io.Serializable;
import java.util.List;
+import org.apache.storm.tuple.ITuple;
+import org.apache.storm.tuple.Values;
/**
* A resultset mapper that
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/BaseCQLStatementTupleMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/BaseCQLStatementTupleMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/BaseCQLStatementTupleMapper.java
index 2a8135d..9cbac15 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/BaseCQLStatementTupleMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/BaseCQLStatementTupleMapper.java
@@ -1,31 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query;
-import org.apache.storm.tuple.ITuple;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
-
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import org.apache.storm.tuple.ITuple;
/**
* Default interface to map a {@link org.apache.storm.tuple.ITuple} to a CQL {@link com.datastax.driver.core.Statement}.
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLResultSetValuesMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLResultSetValuesMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLResultSetValuesMapper.java
index d5495fb..2d50161 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLResultSetValuesMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLResultSetValuesMapper.java
@@ -1,30 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query;
-import org.apache.storm.tuple.ITuple;
-import org.apache.storm.tuple.Values;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
-
import java.io.Serializable;
import java.util.List;
+import org.apache.storm.tuple.ITuple;
+import org.apache.storm.tuple.Values;
/**
*
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLStatementBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLStatementBuilder.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLStatementBuilder.java
index 9ddb58a..c12aad0 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLStatementBuilder.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLStatementBuilder.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLStatementTupleMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLStatementTupleMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLStatementTupleMapper.java
index 1048591..ccc8bc8 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLStatementTupleMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CQLStatementTupleMapper.java
@@ -1,31 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query;
-import org.apache.storm.tuple.ITuple;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
-
import java.io.Serializable;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.storm.tuple.ITuple;
/**
* Default interface to map a {@link org.apache.storm.tuple.ITuple} to a CQL {@link com.datastax.driver.core.Statement}.
@@ -52,7 +45,7 @@ public interface CQLStatementTupleMapper extends Serializable {
@Override
public List<Statement> map(Map<String, Object> conf, Session session, ITuple tuple) {
List<Statement> statements = new LinkedList<>();
- for(CQLStatementBuilder b : builders) {
+ for (CQLStatementBuilder b : builders) {
statements.addAll(b.build().map(conf, session, tuple));
}
return statements;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/Column.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/Column.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/Column.java
index 06643a5..dd94c28 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/Column.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/Column.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query;
import java.io.Serializable;
@@ -35,6 +29,13 @@ public class Column<T> implements Serializable {
this.val = val;
}
+ public static Object[] getVals(List<Column> columns) {
+ List<Object> vals = new ArrayList<>(columns.size());
+ for (Column c : columns)
+ vals.add(c.getVal());
+ return vals.toArray();
+ }
+
public String getColumnName() {
return columnName;
}
@@ -64,11 +65,4 @@ public class Column<T> implements Serializable {
result = 31 * result + (val != null ? val.hashCode() : 0);
return result;
}
-
- public static Object[] getVals(List<Column> columns) {
- List<Object> vals = new ArrayList<>(columns.size());
- for(Column c : columns)
- vals.add(c.getVal());
- return vals.toArray();
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/ContextQuery.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/ContextQuery.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/ContextQuery.java
index 58c7984..bab39eb 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/ContextQuery.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/ContextQuery.java
@@ -1,27 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.cassandra.query;
-import org.apache.storm.tuple.ITuple;
+package org.apache.storm.cassandra.query;
import java.io.Serializable;
import java.util.Map;
+import org.apache.storm.tuple.ITuple;
/**
* This interface may be used to retrieve a cassandra bound query either from storm config
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CqlMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CqlMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CqlMapper.java
index c497f3e..538da8d 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CqlMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/CqlMapper.java
@@ -1,29 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.cassandra.query;
-import org.apache.storm.tuple.ITuple;
-import org.apache.storm.cassandra.query.selector.FieldSelector;
+package org.apache.storm.cassandra.query;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
+import org.apache.storm.cassandra.query.selector.FieldSelector;
+import org.apache.storm.tuple.ITuple;
/**
* Default interface to defines how a storm tuple maps to a list of columns representing a row in a database.
@@ -56,7 +49,7 @@ public interface CqlMapper extends Serializable {
@Override
public List<Column> map(ITuple tuple) {
List<Column> columns = new ArrayList<>(selectors.size());
- for(FieldSelector selector : selectors)
+ for (FieldSelector selector : selectors)
columns.add(selector.select(tuple));
return columns;
}
@@ -78,7 +71,7 @@ public interface CqlMapper extends Serializable {
@Override
public List<Column> map(ITuple tuple) {
List<Column> columns = new ArrayList<>(tuple.size());
- for(String name : tuple.getFields())
+ for (String name : tuple.getFields())
columns.add(new Column(name, tuple.getValueByField(name)));
return columns;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/ObjectMapperOperation.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/ObjectMapperOperation.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/ObjectMapperOperation.java
index 29d7ce9..f6660e5 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/ObjectMapperOperation.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/ObjectMapperOperation.java
@@ -1,20 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.cassandra.query;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/BoundCQLStatementMapperBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/BoundCQLStatementMapperBuilder.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/BoundCQLStatementMapperBuilder.java
index a144830..271ec3d 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/BoundCQLStatementMapperBuilder.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/BoundCQLStatementMapperBuilder.java
@@ -1,23 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query.builder;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
import org.apache.storm.cassandra.query.CQLStatementBuilder;
import org.apache.storm.cassandra.query.ContextQuery;
import org.apache.storm.cassandra.query.CqlMapper;
@@ -26,13 +23,9 @@ import org.apache.storm.cassandra.query.impl.PreparedStatementBinder;
import org.apache.storm.cassandra.query.impl.RoutingKeyGenerator;
import org.apache.storm.cassandra.query.selector.FieldSelector;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.apache.storm.cassandra.query.ContextQuery.*;
+import static org.apache.storm.cassandra.query.ContextQuery.StaticContextQuery;
-public class BoundCQLStatementMapperBuilder implements CQLStatementBuilder<BoundCQLStatementTupleMapper>, Serializable {
+public class BoundCQLStatementMapperBuilder implements CQLStatementBuilder<BoundCQLStatementTupleMapper>, Serializable {
private final ContextQuery contextQuery;
@@ -78,7 +71,7 @@ public class BoundCQLStatementMapperBuilder implements CQLStatementBuilder<Bound
return this;
}
- public final BoundCQLStatementMapperBuilder withRoutingKeys(String...fields) {
+ public final BoundCQLStatementMapperBuilder withRoutingKeys(String... fields) {
this.routingKeys = Arrays.asList(fields);
return this;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/ObjectMapperCqlStatementMapperBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/ObjectMapperCqlStatementMapperBuilder.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/ObjectMapperCqlStatementMapperBuilder.java
index e418468..2044f2d 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/ObjectMapperCqlStatementMapperBuilder.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/ObjectMapperCqlStatementMapperBuilder.java
@@ -1,20 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.cassandra.query.builder;
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/SimpleCQLStatementMapperBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/SimpleCQLStatementMapperBuilder.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/SimpleCQLStatementMapperBuilder.java
index 29ad9c1..0349310 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/SimpleCQLStatementMapperBuilder.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/builder/SimpleCQLStatementMapperBuilder.java
@@ -1,34 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query.builder;
import com.datastax.driver.core.querybuilder.BuiltStatement;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
import org.apache.storm.cassandra.query.CQLStatementBuilder;
import org.apache.storm.cassandra.query.CqlMapper;
import org.apache.storm.cassandra.query.impl.RoutingKeyGenerator;
import org.apache.storm.cassandra.query.impl.SimpleCQLStatementMapper;
import org.apache.storm.cassandra.query.selector.FieldSelector;
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-
/**
* Default class to build {@link org.apache.storm.cassandra.query.impl.SimpleCQLStatementMapper} instance.
*/
@@ -61,7 +54,7 @@ public class SimpleCQLStatementMapperBuilder implements CQLStatementBuilder<Simp
*/
@Override
public SimpleCQLStatementMapper build() {
- return new SimpleCQLStatementMapper(queryString, mapper, (routingKeys != null) ? new RoutingKeyGenerator(routingKeys) : null );
+ return new SimpleCQLStatementMapper(queryString, mapper, (routingKeys != null) ? new RoutingKeyGenerator(routingKeys) : null);
}
/**
@@ -74,7 +67,7 @@ public class SimpleCQLStatementMapperBuilder implements CQLStatementBuilder<Simp
return this;
}
- public final SimpleCQLStatementMapperBuilder withRoutingKeys(String...fields) {
+ public final SimpleCQLStatementMapperBuilder withRoutingKeys(String... fields) {
this.routingKeys = Arrays.asList(fields);
return this;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/BatchCQLStatementTupleMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/BatchCQLStatementTupleMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/BatchCQLStatementTupleMapper.java
index 132981f..8d68a1e 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/BatchCQLStatementTupleMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/BatchCQLStatementTupleMapper.java
@@ -1,33 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query.impl;
-import org.apache.storm.tuple.ITuple;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
-import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
-
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
+import org.apache.storm.tuple.ITuple;
public class BatchCQLStatementTupleMapper implements CQLStatementTupleMapper {
@@ -51,8 +44,8 @@ public class BatchCQLStatementTupleMapper implements CQLStatementTupleMapper {
@Override
public List<Statement> map(Map<String, Object> conf, Session session, ITuple tuple) {
final BatchStatement batch = new BatchStatement(this.type);
- for(CQLStatementTupleMapper m : mappers)
+ for (CQLStatementTupleMapper m : mappers)
batch.addAll(m.map(conf, session, tuple));
- return Arrays.asList((Statement)batch);
+ return Arrays.asList((Statement) batch);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/1a2d131f/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/BoundCQLStatementTupleMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/BoundCQLStatementTupleMapper.java b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/BoundCQLStatementTupleMapper.java
index 04933f2..40e2c34 100644
--- a/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/BoundCQLStatementTupleMapper.java
+++ b/external/storm-cassandra/src/main/java/org/apache/storm/cassandra/query/impl/BoundCQLStatementTupleMapper.java
@@ -1,50 +1,40 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.cassandra.query.impl;
-import org.apache.storm.tuple.ITuple;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
import com.google.common.base.Preconditions;
-import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
-import org.apache.storm.cassandra.query.Column;
-import org.apache.storm.cassandra.query.CqlMapper;
-import org.apache.storm.cassandra.query.ContextQuery;
-
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.storm.cassandra.query.CQLStatementTupleMapper;
+import org.apache.storm.cassandra.query.Column;
+import org.apache.storm.cassandra.query.ContextQuery;
+import org.apache.storm.cassandra.query.CqlMapper;
+import org.apache.storm.tuple.ITuple;
public class BoundCQLStatementTupleMapper implements CQLStatementTupleMapper {
private final ContextQuery contextQuery;
private final CqlMapper mapper;
-
- private Map<String, PreparedStatement> cache = new HashMap<>();
-
private final RoutingKeyGenerator rkGenerator;
-
private final PreparedStatementBinder binder;
+ private Map<String, PreparedStatement> cache = new HashMap<>();
/**
* Creates a new {@link BoundCQLStatementTupleMapper} instance.
@@ -55,7 +45,8 @@ public class BoundCQLStatementTupleMapper implements CQLStatementTupleMapper {
* @param rkGenerator
* @param binder
*/
- public BoundCQLStatementTupleMapper(ContextQuery contextQuery, CqlMapper mapper, RoutingKeyGenerator rkGenerator, PreparedStatementBinder binder) {
+ public BoundCQLStatementTupleMapper(ContextQuery contextQuery, CqlMapper mapper, RoutingKeyGenerator rkGenerator,
+ PreparedStatementBinder binder) {
Preconditions.checkNotNull(contextQuery, "ContextQuery must not be null");
Preconditions.checkNotNull(mapper, "Mapper should not be null");
this.contextQuery = contextQuery;
@@ -74,12 +65,13 @@ public class BoundCQLStatementTupleMapper implements CQLStatementTupleMapper {
final String query = contextQuery.resolves(config, tuple);
PreparedStatement statement = getPreparedStatement(session, query);
- if(hasRoutingKeys()) {
+ if (hasRoutingKeys()) {
List<ByteBuffer> keys = rkGenerator.getRoutingKeys(tuple);
- if( keys.size() == 1)
+ if (keys.size() == 1) {
statement.setRoutingKey(keys.get(0));
- else
+ } else {
statement.setRoutingKey(keys.toArray(new ByteBuffer[keys.size()]));
+ }
}
return Arrays.asList((Statement) this.binder.apply(statement, columns));
@@ -97,7 +89,7 @@ public class BoundCQLStatementTupleMapper implements CQLStatementTupleMapper {
*/
private PreparedStatement getPreparedStatement(Session session, String query) {
PreparedStatement statement = cache.get(query);
- if( statement == null) {
+ if (statement == null) {
statement = session.prepare(query);
cache.put(query, statement);
}
[60/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-mqtt
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-mqtt
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f7f3524d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f7f3524d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f7f3524d
Branch: refs/heads/master
Commit: f7f3524d1401d76e1779c823391dce46e58acf48
Parents: 6d20c4a
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 23:32:01 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 02:32:42 2018 -0400
----------------------------------------------------------------------
external/storm-mqtt/pom.xml | 2 +-
.../java/org/apache/storm/mqtt/MqttLogger.java | 20 ++---
.../java/org/apache/storm/mqtt/MqttMessage.java | 26 +++---
.../apache/storm/mqtt/MqttMessageMapper.java | 22 ++---
.../org/apache/storm/mqtt/MqttTupleMapper.java | 24 ++----
.../org/apache/storm/mqtt/bolt/MqttBolt.java | 36 ++++----
.../apache/storm/mqtt/common/MqttOptions.java | 43 +++++-----
.../apache/storm/mqtt/common/MqttPublisher.java | 28 +++----
.../org/apache/storm/mqtt/common/MqttUtils.java | 34 ++++----
.../org/apache/storm/mqtt/common/SslUtils.java | 36 ++++----
.../mqtt/mappers/ByteArrayMessageMapper.java | 23 ++----
.../storm/mqtt/mappers/StringMessageMapper.java | 23 ++----
.../apache/storm/mqtt/spout/AckableMessage.java | 49 ++++++-----
.../org/apache/storm/mqtt/spout/MqttSpout.java | 86 +++++++++-----------
.../storm/mqtt/ssl/DefaultKeyStoreLoader.java | 25 +++---
.../apache/storm/mqtt/ssl/KeyStoreLoader.java | 23 +++---
.../storm/mqtt/trident/MqttPublishFunction.java | 33 +++-----
.../storm/mqtt/StormMqttIntegrationTest.java | 75 ++++++++---------
18 files changed, 254 insertions(+), 354 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/pom.xml b/external/storm-mqtt/pom.xml
index 6c5c48c..703b5da 100644
--- a/external/storm-mqtt/pom.xml
+++ b/external/storm-mqtt/pom.xml
@@ -127,7 +127,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>158</maxAllowedViolations>
+ <maxAllowedViolations>39</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttLogger.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttLogger.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttLogger.java
index 3af73fd..3644d16 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttLogger.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttLogger.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt;
import org.fusesource.mqtt.client.Tracer;
-import org.fusesource.mqtt.codec.MQTTFrame;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttMessage.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttMessage.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttMessage.java
index 5436dda..90401ef 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttMessage.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttMessage.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt;
/**
@@ -27,15 +22,16 @@ public class MqttMessage {
private byte[] message;
- public MqttMessage(String topic, byte[] payload){
+ public MqttMessage(String topic, byte[] payload) {
this.topic = topic;
this.message = payload;
}
- public byte[] getMessage(){
+
+ public byte[] getMessage() {
return this.message;
}
- public String getTopic(){
+ public String getTopic() {
return this.topic;
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttMessageMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttMessageMapper.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttMessageMapper.java
index c6173f4..caeb605 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttMessageMapper.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttMessageMapper.java
@@ -1,27 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt;
+import java.io.Serializable;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
-import java.io.Serializable;
-
/**
* Represents an object that can be converted to a Storm Tuple from an AckableMessage,
* given a MQTT Topic Name and a byte array payload.
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttTupleMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttTupleMapper.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttTupleMapper.java
index c46c069..d3762b0 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttTupleMapper.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/MqttTupleMapper.java
@@ -1,31 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.mqtt;
+package org.apache.storm.mqtt;
-import org.apache.storm.tuple.ITuple;
import java.io.Serializable;
+import org.apache.storm.tuple.ITuple;
/**
* Given a Tuple, converts it to an MQTT message.
*/
-public interface MqttTupleMapper extends Serializable{
+public interface MqttTupleMapper extends Serializable {
/**
* Converts a Tuple to a MqttMessage
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/bolt/MqttBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/bolt/MqttBolt.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/bolt/MqttBolt.java
index b8f5068..745607b 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/bolt/MqttBolt.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/bolt/MqttBolt.java
@@ -1,41 +1,33 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt.bolt;
+import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.mqtt.MqttMessage;
-import org.apache.storm.mqtt.common.MqttOptions;
import org.apache.storm.mqtt.MqttTupleMapper;
+import org.apache.storm.mqtt.common.MqttOptions;
import org.apache.storm.mqtt.common.MqttPublisher;
import org.apache.storm.mqtt.common.SslUtils;
import org.apache.storm.mqtt.ssl.KeyStoreLoader;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.topology.base.BaseTickTupleAwareRichBolt;
import org.apache.storm.tuple.Tuple;
-import org.apache.storm.utils.TupleUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
-
public class MqttBolt extends BaseTickTupleAwareRichBolt {
private static final Logger LOG = LoggerFactory.getLogger(MqttBolt.class);
@@ -48,19 +40,19 @@ public class MqttBolt extends BaseTickTupleAwareRichBolt {
private transient String topologyName;
- public MqttBolt(MqttOptions options, MqttTupleMapper mapper){
+ public MqttBolt(MqttOptions options, MqttTupleMapper mapper) {
this(options, mapper, null, false);
}
- public MqttBolt(MqttOptions options, MqttTupleMapper mapper, boolean retain){
+ public MqttBolt(MqttOptions options, MqttTupleMapper mapper, boolean retain) {
this(options, mapper, null, retain);
}
- public MqttBolt(MqttOptions options, MqttTupleMapper mapper, KeyStoreLoader keyStoreLoader){
+ public MqttBolt(MqttOptions options, MqttTupleMapper mapper, KeyStoreLoader keyStoreLoader) {
this(options, mapper, keyStoreLoader, false);
}
- public MqttBolt(MqttOptions options, MqttTupleMapper mapper, KeyStoreLoader keyStoreLoader, boolean retain){
+ public MqttBolt(MqttOptions options, MqttTupleMapper mapper, KeyStoreLoader keyStoreLoader, boolean retain) {
this.options = options;
this.mapper = mapper;
this.retain = retain;
@@ -74,7 +66,7 @@ public class MqttBolt extends BaseTickTupleAwareRichBolt {
@Override
public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
this.collector = collector;
- this.topologyName = (String)conf.get(Config.TOPOLOGY_NAME);
+ this.topologyName = (String) conf.get(Config.TOPOLOGY_NAME);
this.publisher = new MqttPublisher(this.options, this.keyStoreLoader, this.retain);
try {
this.publisher.connectMqtt(this.topologyName + "-" + context.getThisComponentId() + "-" + context.getThisTaskId());
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttOptions.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttOptions.java
index 2b09d6e..1e9c925 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttOptions.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttOptions.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt.common;
import java.io.Serializable;
@@ -34,7 +29,7 @@ public class MqttOptions implements Serializable {
private boolean willRetain = false;
private long reconnectDelay = 10;
- private long reconnectDelayMax = 30*1000;
+ private long reconnectDelayMax = 30 * 1000;
private double reconnectBackOffMultiplier = 2.0f;
private long reconnectAttemptsMax = -1;
private long connectAttemptsMax = -1;
@@ -205,7 +200,7 @@ public class MqttOptions implements Serializable {
this.password = password;
}
- public int getQos(){
+ public int getQos() {
return this.qos;
}
@@ -213,14 +208,14 @@ public class MqttOptions implements Serializable {
* Sets the quality of service to use for MQTT messages. Defaults to 1 (at least once).
* @param qos
*/
- public void setQos(int qos){
- if(qos < 0 || qos > 2){
+ public void setQos(int qos) {
+ if (qos < 0 || qos > 2) {
throw new IllegalArgumentException("MQTT QoS must be >= 0 and <= 2");
}
this.qos = qos;
}
- public int getWillQos(){
+ public int getWillQos() {
return this.willQos;
}
@@ -229,14 +224,14 @@ public class MqttOptions implements Serializable {
*
* @param qos
*/
- public void setWillQos(int qos){
- if(qos < 0 || qos > 2){
+ public void setWillQos(int qos) {
+ if (qos < 0 || qos > 2) {
throw new IllegalArgumentException("MQTT Will QoS must be >= 0 and <= 2");
}
this.willQos = qos;
}
- public boolean getWillRetain(){
+ public boolean getWillRetain() {
return this.willRetain;
}
@@ -244,7 +239,7 @@ public class MqttOptions implements Serializable {
* Set to true if you want the Will message to be published with the retain option.
* @param retain
*/
- public void setWillRetain(boolean retain){
+ public void setWillRetain(boolean retain) {
this.willRetain = retain;
}
@@ -277,12 +272,12 @@ public class MqttOptions implements Serializable {
return this;
}
- public Builder willRetain(boolean retain){
+ public Builder willRetain(boolean retain) {
this.options.willRetain = retain;
return this;
}
- public Builder willQos(int qos){
+ public Builder willQos(int qos) {
this.options.setWillQos(qos);
return this;
}
@@ -322,7 +317,7 @@ public class MqttOptions implements Serializable {
return this;
}
- public Builder qos(int qos){
+ public Builder qos(int qos) {
this.options.setQos(qos);
return this;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttPublisher.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttPublisher.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttPublisher.java
index 9b36b78..5ea1da8 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttPublisher.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttPublisher.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt.common;
-import org.apache.storm.mqtt.MqttLogger;
import org.apache.storm.mqtt.MqttMessage;
import org.apache.storm.mqtt.ssl.KeyStoreLoader;
import org.fusesource.mqtt.client.BlockingConnection;
@@ -27,8 +21,6 @@ import org.fusesource.mqtt.client.QoS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.URI;
-
public class MqttPublisher {
private static final Logger LOG = LoggerFactory.getLogger(MqttPublisher.class);
@@ -39,15 +31,15 @@ public class MqttPublisher {
private boolean retain = false;
- public MqttPublisher(MqttOptions options){
+ public MqttPublisher(MqttOptions options) {
this(options, null, false);
}
- public MqttPublisher(MqttOptions options, boolean retain){
+ public MqttPublisher(MqttOptions options, boolean retain) {
this(options, null, retain);
}
- public MqttPublisher(MqttOptions options, KeyStoreLoader keyStoreLoader, boolean retain){
+ public MqttPublisher(MqttOptions options, KeyStoreLoader keyStoreLoader, boolean retain) {
this.retain = retain;
this.options = options;
this.keyStoreLoader = keyStoreLoader;
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttUtils.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttUtils.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttUtils.java
index 4ca0145..4995fd0 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttUtils.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/MqttUtils.java
@@ -1,23 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt.common;
+import java.net.URI;
import org.apache.storm.mqtt.MqttLogger;
import org.apache.storm.mqtt.ssl.KeyStoreLoader;
import org.fusesource.mqtt.client.MQTT;
@@ -25,16 +21,14 @@ import org.fusesource.mqtt.client.QoS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.URI;
-
public class MqttUtils {
private static final Logger LOG = LoggerFactory.getLogger(MqttUtils.class);
- private MqttUtils(){}
+ private MqttUtils() {}
- public static QoS qosFromInt(int i){
+ public static QoS qosFromInt(int i) {
QoS qos = null;
- switch(i) {
+ switch (i) {
case 0:
qos = QoS.AT_MOST_ONCE;
break;
@@ -52,13 +46,13 @@ public class MqttUtils {
public static MQTT configureClient(MqttOptions options, String clientId, KeyStoreLoader keyStoreLoader)
- throws Exception{
+ throws Exception {
MQTT client = new MQTT();
URI uri = URI.create(options.getUrl());
client.setHost(uri);
- if(!uri.getScheme().toLowerCase().equals("tcp")){
+ if (!uri.getScheme().toLowerCase().equals("tcp")) {
client.setSslContext(SslUtils.sslContext(uri.getScheme(), keyStoreLoader));
}
client.setClientId(clientId);
@@ -76,7 +70,7 @@ public class MqttUtils {
client.setPassword(options.getPassword());
client.setTracer(new MqttLogger());
- if(options.getWillTopic() != null && options.getWillPayload() != null){
+ if (options.getWillTopic() != null && options.getWillPayload() != null) {
QoS qos = MqttUtils.qosFromInt(options.getWillQos());
client.setWillQos(qos);
client.setWillTopic(options.getWillTopic());
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/SslUtils.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/SslUtils.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/SslUtils.java
index 681fc1d..6ea3d0c 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/SslUtils.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/common/SslUtils.java
@@ -1,44 +1,38 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.mqtt.common;
+package org.apache.storm.mqtt.common;
-import org.apache.storm.mqtt.ssl.KeyStoreLoader;
+import java.net.URI;
+import java.security.KeyStore;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
-import java.net.URI;
-import java.security.KeyStore;
+import org.apache.storm.mqtt.ssl.KeyStoreLoader;
public class SslUtils {
- private SslUtils(){}
+ private SslUtils() {}
- public static void checkSslConfig(String url, KeyStoreLoader loader){
+ public static void checkSslConfig(String url, KeyStoreLoader loader) {
URI uri = URI.create(url);
String scheme = uri.getScheme().toLowerCase();
- if(!(scheme.equals("tcp") || scheme.startsWith("tls") || scheme.startsWith("ssl"))){
+ if (!(scheme.equals("tcp") || scheme.startsWith("tls") || scheme.startsWith("ssl"))) {
throw new IllegalArgumentException("Unrecognized URI scheme: " + scheme);
}
- if(!scheme.equalsIgnoreCase("tcp") && loader == null){
+ if (!scheme.equalsIgnoreCase("tcp") && loader == null) {
throw new IllegalStateException("A TLS/SSL MQTT URL was specified, but no KeyStoreLoader configured. " +
- "A KeyStoreLoader implementation is required when using TLS/SSL.");
+ "A KeyStoreLoader implementation is required when using TLS/SSL.");
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/mappers/ByteArrayMessageMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/mappers/ByteArrayMessageMapper.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/mappers/ByteArrayMessageMapper.java
index a19fce4..1dd943d 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/mappers/ByteArrayMessageMapper.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/mappers/ByteArrayMessageMapper.java
@@ -1,26 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt.mappers;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Values;
import org.apache.storm.mqtt.MqttMessage;
import org.apache.storm.mqtt.MqttMessageMapper;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Values;
public class ByteArrayMessageMapper implements MqttMessageMapper {
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/mappers/StringMessageMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/mappers/StringMessageMapper.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/mappers/StringMessageMapper.java
index e5f309b..341e52d 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/mappers/StringMessageMapper.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/mappers/StringMessageMapper.java
@@ -1,26 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt.mappers;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Values;
import org.apache.storm.mqtt.MqttMessage;
import org.apache.storm.mqtt.MqttMessageMapper;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Values;
/**
* Given a String topic and byte[] message, emits a tuple with fields
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/spout/AckableMessage.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/spout/AckableMessage.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/spout/AckableMessage.java
index 5c3867e..0b91442 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/spout/AckableMessage.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/spout/AckableMessage.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt.spout;
import org.apache.commons.lang.builder.EqualsBuilder;
@@ -31,41 +26,45 @@ class AckableMessage {
private byte[] message;
private Runnable ack;
- AckableMessage(String topic, byte[] message, Runnable ack){
+ AckableMessage(String topic, byte[] message, Runnable ack) {
this.topic = topic;
this.message = message;
this.ack = ack;
}
- public MqttMessage getMessage(){
+ public MqttMessage getMessage() {
return new MqttMessage(this.topic, this.message);
}
@Override
public int hashCode() {
return new HashCodeBuilder(71, 123)
- .append(this.topic)
- .append(this.message)
- .toHashCode();
+ .append(this.topic)
+ .append(this.message)
+ .toHashCode();
}
@Override
public boolean equals(Object obj) {
- if (obj == null) { return false; }
- if (obj == this) { return true; }
+ if (obj == null) {
+ return false;
+ }
+ if (obj == this) {
+ return true;
+ }
if (obj.getClass() != getClass()) {
return false;
}
- AckableMessage tm = (AckableMessage)obj;
+ AckableMessage tm = (AckableMessage) obj;
return new EqualsBuilder()
- .appendSuper(super.equals(obj))
- .append(this.topic, tm.topic)
- .append(this.message, tm.message)
- .isEquals();
+ .appendSuper(super.equals(obj))
+ .append(this.topic, tm.topic)
+ .append(this.message, tm.message)
+ .isEquals();
}
- Runnable ack(){
+ Runnable ack() {
return this.ack;
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/spout/MqttSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/spout/MqttSpout.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/spout/MqttSpout.java
index df6c6fd..268a695 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/spout/MqttSpout.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/spout/MqttSpout.java
@@ -1,32 +1,31 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt.spout;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
import org.apache.storm.Config;
-import org.apache.storm.spout.SpoutOutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.IRichSpout;
-import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.mqtt.MqttMessageMapper;
import org.apache.storm.mqtt.common.MqttOptions;
import org.apache.storm.mqtt.common.MqttUtils;
import org.apache.storm.mqtt.common.SslUtils;
import org.apache.storm.mqtt.ssl.KeyStoreLoader;
+import org.apache.storm.spout.SpoutOutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.IRichSpout;
+import org.apache.storm.topology.OutputFieldsDeclarer;
import org.fusesource.hawtbuf.Buffer;
import org.fusesource.hawtbuf.UTF8Buffer;
import org.fusesource.mqtt.client.Callback;
@@ -38,55 +37,45 @@ import org.fusesource.mqtt.client.Topic;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.LinkedBlockingQueue;
-
public class MqttSpout implements IRichSpout, Listener {
private static final Logger LOG = LoggerFactory.getLogger(MqttSpout.class);
-
- private String topologyName;
-
-
- private CallbackConnection connection;
-
protected transient SpoutOutputCollector collector;
protected transient TopologyContext context;
protected transient LinkedBlockingQueue<AckableMessage> incoming;
protected transient HashMap<Long, AckableMessage> pending;
- private transient Map<String, Object> conf;
protected MqttMessageMapper type;
protected MqttOptions options;
protected KeyStoreLoader keyStoreLoader;
-
+ private String topologyName;
+ private CallbackConnection connection;
+ private transient Map<String, Object> conf;
private boolean mqttConnected = false;
private boolean mqttConnectFailed = false;
private Long sequence = Long.MIN_VALUE;
- private Long nextId(){
- this.sequence++;
- if(this.sequence == Long.MAX_VALUE){
- this.sequence = Long.MIN_VALUE;
- }
- return this.sequence;
- }
-
- protected MqttSpout(){}
+ protected MqttSpout() {}
- public MqttSpout(MqttMessageMapper type, MqttOptions options){
+ public MqttSpout(MqttMessageMapper type, MqttOptions options) {
this(type, options, null);
}
- public MqttSpout(MqttMessageMapper type, MqttOptions options, KeyStoreLoader keyStoreLoader){
+ public MqttSpout(MqttMessageMapper type, MqttOptions options, KeyStoreLoader keyStoreLoader) {
this.type = type;
this.options = options;
this.keyStoreLoader = keyStoreLoader;
SslUtils.checkSslConfig(this.options.getUrl(), this.keyStoreLoader);
}
+ private Long nextId() {
+ this.sequence++;
+ if (this.sequence == Long.MAX_VALUE) {
+ this.sequence = Long.MIN_VALUE;
+ }
+ return this.sequence;
+ }
+
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(this.type.outputFields());
}
@@ -96,7 +85,7 @@ public class MqttSpout implements IRichSpout, Listener {
}
public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
- this.topologyName = (String)conf.get(Config.TOPOLOGY_NAME);
+ this.topologyName = (String) conf.get(Config.TOPOLOGY_NAME);
this.collector = collector;
this.context = context;
@@ -116,23 +105,23 @@ public class MqttSpout implements IRichSpout, Listener {
private void connectMqtt() throws Exception {
String clientId = this.topologyName + "-" + this.context.getThisComponentId() + "-" +
- this.context.getThisTaskId();
+ this.context.getThisTaskId();
MQTT client = MqttUtils.configureClient(this.options, clientId, this.keyStoreLoader);
this.connection = client.callbackConnection();
this.connection.listener(this);
this.connection.connect(new ConnectCallback());
- while(!this.mqttConnected && !this.mqttConnectFailed){
+ while (!this.mqttConnected && !this.mqttConnectFailed) {
LOG.info("Waiting for connection...");
Thread.sleep(500);
}
- if(this.mqttConnected){
+ if (this.mqttConnected) {
List<String> topicList = this.options.getTopics();
Topic[] topics = new Topic[topicList.size()];
QoS qos = MqttUtils.qosFromInt(this.options.getQos());
- for(int i = 0;i < topicList.size();i++){
+ for (int i = 0; i < topicList.size(); i++) {
topics[i] = new Topic(topicList.get(i), qos);
}
connection.subscribe(topics, new SubscribeCallback());
@@ -140,7 +129,6 @@ public class MqttSpout implements IRichSpout, Listener {
}
-
public void close() {
this.connection.disconnect(new DisconnectCallback());
}
@@ -161,7 +149,7 @@ public class MqttSpout implements IRichSpout, Listener {
*/
public void nextTuple() {
AckableMessage tm = this.incoming.poll();
- if(tm != null){
+ if (tm != null) {
Long id = nextId();
this.collector.emit(this.type.toValues(tm.getMessage()), id);
this.pending.put(id, tm);
@@ -237,7 +225,7 @@ public class MqttSpout implements IRichSpout, Listener {
}
// ################# Subscribe Callback Implementation ######################
- private class SubscribeCallback implements Callback<byte[]>{
+ private class SubscribeCallback implements Callback<byte[]> {
public void onSuccess(byte[] qos) {
LOG.info("Subscripton sucessful.");
}
@@ -249,7 +237,7 @@ public class MqttSpout implements IRichSpout, Listener {
}
// ################# Subscribe Callback Implementation ######################
- private class DisconnectCallback implements Callback<Void>{
+ private class DisconnectCallback implements Callback<Void> {
public void onSuccess(Void aVoid) {
LOG.info("MQTT Disconnect successful.");
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/ssl/DefaultKeyStoreLoader.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/ssl/DefaultKeyStoreLoader.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/ssl/DefaultKeyStoreLoader.java
index 8bca407..14c3c6c 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/ssl/DefaultKeyStoreLoader.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/ssl/DefaultKeyStoreLoader.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt.ssl;
import java.io.FileInputStream;
@@ -37,7 +32,7 @@ public class DefaultKeyStoreLoader implements KeyStoreLoader {
*
* @param keystore path to keystore file
*/
- public DefaultKeyStoreLoader(String keystore){
+ public DefaultKeyStoreLoader(String keystore) {
this.ksFile = keystore;
}
@@ -48,7 +43,7 @@ public class DefaultKeyStoreLoader implements KeyStoreLoader {
* @param keystore path to keystore file
* @param truststore path to truststore file
*/
- public DefaultKeyStoreLoader(String keystore, String truststore){
+ public DefaultKeyStoreLoader(String keystore, String truststore) {
this.ksFile = keystore;
this.tsFile = truststore;
}
@@ -73,7 +68,7 @@ public class DefaultKeyStoreLoader implements KeyStoreLoader {
@Override
public InputStream trustStoreInputStream() throws FileNotFoundException {
// if no truststore file, assume the truststore is the keystore.
- if(this.tsFile == null){
+ if (this.tsFile == null) {
return new FileInputStream(this.ksFile);
} else {
return new FileInputStream(this.tsFile);
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/ssl/KeyStoreLoader.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/ssl/KeyStoreLoader.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/ssl/KeyStoreLoader.java
index 297efcc..5630ff8 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/ssl/KeyStoreLoader.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/ssl/KeyStoreLoader.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt.ssl;
import java.io.IOException;
@@ -28,8 +23,12 @@ import java.io.Serializable;
public interface KeyStoreLoader extends Serializable {
String keyStorePassword();
+
String trustStorePassword();
+
String keyPassword();
+
InputStream keyStoreInputStream() throws IOException;
+
InputStream trustStoreInputStream() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/trident/MqttPublishFunction.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/trident/MqttPublishFunction.java b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/trident/MqttPublishFunction.java
index 5a821c6..329cff5 100644
--- a/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/trident/MqttPublishFunction.java
+++ b/external/storm-mqtt/src/main/java/org/apache/storm/mqtt/trident/MqttPublishFunction.java
@@ -1,31 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt.trident;
+import java.util.Map;
import org.apache.storm.Config;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.topology.FailedException;
import org.apache.storm.mqtt.MqttMessage;
-import org.apache.storm.mqtt.common.MqttOptions;
import org.apache.storm.mqtt.MqttTupleMapper;
+import org.apache.storm.mqtt.common.MqttOptions;
import org.apache.storm.mqtt.common.MqttPublisher;
import org.apache.storm.mqtt.common.SslUtils;
import org.apache.storm.mqtt.ssl.KeyStoreLoader;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.topology.FailedException;
import org.apache.storm.trident.operation.BaseFunction;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.operation.TridentOperationContext;
@@ -33,9 +29,6 @@ import org.apache.storm.trident.tuple.TridentTuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
public class MqttPublishFunction extends BaseFunction {
private static final Logger LOG = LoggerFactory.getLogger(MqttPublishFunction.class);
private MqttTupleMapper mapper;
@@ -47,7 +40,7 @@ public class MqttPublishFunction extends BaseFunction {
private transient String topologyName;
- public MqttPublishFunction(MqttOptions options, MqttTupleMapper mapper, KeyStoreLoader keyStoreLoader, boolean retain){
+ public MqttPublishFunction(MqttOptions options, MqttTupleMapper mapper, KeyStoreLoader keyStoreLoader, boolean retain) {
this.options = options;
this.mapper = mapper;
this.retain = retain;
@@ -61,7 +54,7 @@ public class MqttPublishFunction extends BaseFunction {
@Override
public void prepare(Map<String, Object> conf, TridentOperationContext context) {
- this.topologyName = (String)conf.get(Config.TOPOLOGY_NAME);
+ this.topologyName = (String) conf.get(Config.TOPOLOGY_NAME);
this.publisher = new MqttPublisher(this.options, this.keyStoreLoader, this.retain);
try {
this.publisher.connectMqtt(this.topologyName + "-" + context.getPartitionIndex());
http://git-wip-us.apache.org/repos/asf/storm/blob/f7f3524d/external/storm-mqtt/src/test/java/org/apache/storm/mqtt/StormMqttIntegrationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-mqtt/src/test/java/org/apache/storm/mqtt/StormMqttIntegrationTest.java b/external/storm-mqtt/src/test/java/org/apache/storm/mqtt/StormMqttIntegrationTest.java
index 0dd4d73..ba214eb 100644
--- a/external/storm-mqtt/src/test/java/org/apache/storm/mqtt/StormMqttIntegrationTest.java
+++ b/external/storm-mqtt/src/test/java/org/apache/storm/mqtt/StormMqttIntegrationTest.java
@@ -1,35 +1,33 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.mqtt;
+import java.io.Serializable;
+import java.net.URI;
+import java.util.Arrays;
+import org.apache.activemq.broker.BrokerService;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.LocalCluster.LocalTopology;
import org.apache.storm.generated.StormTopology;
-import org.apache.storm.testing.IntegrationTest;
-import org.apache.storm.topology.TopologyBuilder;
-import org.apache.storm.tuple.ITuple;
-import org.apache.activemq.broker.BrokerService;
import org.apache.storm.mqtt.bolt.MqttBolt;
import org.apache.storm.mqtt.common.MqttOptions;
import org.apache.storm.mqtt.common.MqttPublisher;
import org.apache.storm.mqtt.mappers.StringMessageMapper;
import org.apache.storm.mqtt.spout.MqttSpout;
+import org.apache.storm.testing.IntegrationTest;
+import org.apache.storm.topology.TopologyBuilder;
+import org.apache.storm.tuple.ITuple;
import org.fusesource.mqtt.client.BlockingConnection;
import org.fusesource.mqtt.client.MQTT;
import org.fusesource.mqtt.client.Message;
@@ -43,33 +41,14 @@ import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Serializable;
-import java.net.URI;
-import java.util.Arrays;
-
@Category(IntegrationTest.class)
-public class StormMqttIntegrationTest implements Serializable{
+public class StormMqttIntegrationTest implements Serializable {
private static final Logger LOG = LoggerFactory.getLogger(StormMqttIntegrationTest.class);
- private static BrokerService broker;
- static boolean spoutActivated = false;
-
private static final String TEST_TOPIC = "/mqtt-topology";
private static final String RESULT_TOPIC = "/integration-result";
private static final String RESULT_PAYLOAD = "Storm MQTT Spout";
-
- public static class TestSpout extends MqttSpout{
- public TestSpout(MqttMessageMapper type, MqttOptions options){
- super(type, options);
- }
-
- @Override
- public void activate() {
- super.activate();
- LOG.info("Spout activated.");
- spoutActivated = true;
- }
- }
-
+ static boolean spoutActivated = false;
+ private static BrokerService broker;
@AfterClass
public static void cleanup() throws Exception {
@@ -86,7 +65,6 @@ public class StormMqttIntegrationTest implements Serializable{
LOG.debug("MQTT broker started");
}
-
@Test
public void testMqttTopology() throws Exception {
MQTT client = new MQTT();
@@ -98,14 +76,14 @@ public class StormMqttIntegrationTest implements Serializable{
client.setCleanSession(false);
BlockingConnection connection = client.blockingConnection();
connection.connect();
- Topic[] topics = {new Topic("/integration-result", QoS.AT_LEAST_ONCE)};
+ Topic[] topics = { new Topic("/integration-result", QoS.AT_LEAST_ONCE) };
byte[] qoses = connection.subscribe(topics);
try (LocalCluster cluster = new LocalCluster();
LocalTopology topo = cluster.submitTopology("test", new Config(), buildMqttTopology());) {
LOG.info("topology started");
- while(!spoutActivated) {
+ while (!spoutActivated) {
Thread.sleep(500);
}
@@ -128,7 +106,7 @@ public class StormMqttIntegrationTest implements Serializable{
}
}
- public StormTopology buildMqttTopology(){
+ public StormTopology buildMqttTopology() {
TopologyBuilder builder = new TopologyBuilder();
MqttOptions options = new MqttOptions();
@@ -150,4 +128,17 @@ public class StormMqttIntegrationTest implements Serializable{
return builder.createTopology();
}
+ public static class TestSpout extends MqttSpout {
+ public TestSpout(MqttMessageMapper type, MqttOptions options) {
+ super(type, options);
+ }
+
+ @Override
+ public void activate() {
+ super.activate();
+ LOG.info("Spout activated.");
+ spoutActivated = true;
+ }
+ }
+
}
[07/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceNameNormalizer.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceNameNormalizer.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceNameNormalizer.java
index fc9182d..57ee348 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceNameNormalizer.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/normalization/ResourceNameNormalizer.java
@@ -54,11 +54,11 @@ public class ResourceNameNormalizer {
return new HashMap<>();
}
return new HashMap<>(resourceMap.entrySet().stream()
- .collect(Collectors.toMap(
- //Map the key if needed
- (e) -> resourceNameMapping.getOrDefault(e.getKey(), e.getKey()),
- //Map the value
- (e) -> e.getValue().doubleValue())));
+ .collect(Collectors.toMap(
+ //Map the key if needed
+ (e) -> resourceNameMapping.getOrDefault(e.getKey(), e.getKey()),
+ //Map the value
+ (e) -> e.getValue().doubleValue())));
}
public Map<String, String> getResourceNameMapping() {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/DefaultSchedulingPriorityStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/DefaultSchedulingPriorityStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/DefaultSchedulingPriorityStrategy.java
index 8b401aa..2e1ca79 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/DefaultSchedulingPriorityStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/DefaultSchedulingPriorityStrategy.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource.strategies.priority;
@@ -33,12 +27,44 @@ import org.slf4j.LoggerFactory;
public class DefaultSchedulingPriorityStrategy implements ISchedulingPriorityStrategy {
private static final Logger LOG = LoggerFactory.getLogger(DefaultSchedulingPriorityStrategy.class);
+ protected SimulatedUser getSimulatedUserFor(User u, ISchedulingState cluster) {
+ return new SimulatedUser(u, cluster);
+ }
+
+ @Override
+ public List<TopologyDetails> getOrderedTopologies(ISchedulingState cluster, Map<String, User> userMap) {
+ double cpuAvail = cluster.getClusterTotalCpuResource();
+ double memAvail = cluster.getClusterTotalMemoryResource();
+
+ List<TopologyDetails> allUserTopologies = new ArrayList<>();
+ List<SimulatedUser> users = new ArrayList<>();
+ for (User u : userMap.values()) {
+ users.add(getSimulatedUserFor(u, cluster));
+ }
+ while (!users.isEmpty()) {
+ Collections.sort(users, new SimulatedUserComparator(cpuAvail, memAvail));
+ SimulatedUser u = users.get(0);
+ TopologyDetails td = u.getNextHighest();
+ if (td == null) {
+ users.remove(0);
+ } else {
+ double score = u.getScore(cpuAvail, memAvail);
+ td = u.simScheduleNextHighest();
+ LOG.info("SIM Scheduling {} with score of {}", td.getId(), score);
+ cpuAvail -= td.getTotalRequestedCpu();
+ memAvail -= (td.getTotalRequestedMemOffHeap() + td.getTotalRequestedMemOnHeap());
+ allUserTopologies.add(td);
+ }
+ }
+ return allUserTopologies;
+ }
+
protected static class SimulatedUser {
+ public final double guaranteedCpu;
+ public final double guaranteedMemory;
protected final LinkedList<TopologyDetails> tds = new LinkedList<>();
private double assignedCpu = 0.0;
private double assignedMemory = 0.0;
- public final double guaranteedCpu;
- public final double guaranteedMemory;
public SimulatedUser(User other, ISchedulingState cluster) {
tds.addAll(cluster.getTopologies().getTopologiesOwnedBy(other.getId()));
@@ -82,8 +108,8 @@ public class DefaultSchedulingPriorityStrategy implements ISchedulingPriorityStr
}
double wouldBeCpu = assignedCpu + td.getTotalRequestedCpu();
double wouldBeMem = assignedMemory + td.getTotalRequestedMemOffHeap() + td.getTotalRequestedMemOnHeap();
- double cpuScore = (wouldBeCpu - guaranteedCpu)/availableCpu;
- double memScore = (wouldBeMem - guaranteedMemory)/availableMemory;
+ double cpuScore = (wouldBeCpu - guaranteedCpu) / availableCpu;
+ double memScore = (wouldBeMem - guaranteedMemory) / availableMemory;
return Math.max(cpuScore, memScore);
}
@@ -94,38 +120,6 @@ public class DefaultSchedulingPriorityStrategy implements ISchedulingPriorityStr
}
- protected SimulatedUser getSimulatedUserFor(User u, ISchedulingState cluster) {
- return new SimulatedUser(u, cluster);
- }
-
- @Override
- public List<TopologyDetails> getOrderedTopologies(ISchedulingState cluster, Map<String, User> userMap) {
- double cpuAvail = cluster.getClusterTotalCpuResource();
- double memAvail = cluster.getClusterTotalMemoryResource();
-
- List<TopologyDetails> allUserTopologies = new ArrayList<>();
- List<SimulatedUser> users = new ArrayList<>();
- for (User u : userMap.values()) {
- users.add(getSimulatedUserFor(u, cluster));
- }
- while (!users.isEmpty()) {
- Collections.sort(users, new SimulatedUserComparator(cpuAvail, memAvail));
- SimulatedUser u = users.get(0);
- TopologyDetails td = u.getNextHighest();
- if (td == null) {
- users.remove(0);
- } else {
- double score = u.getScore(cpuAvail, memAvail);
- td = u.simScheduleNextHighest();
- LOG.info("SIM Scheduling {} with score of {}", td.getId(), score);
- cpuAvail -= td.getTotalRequestedCpu();
- memAvail -= (td.getTotalRequestedMemOffHeap() + td.getTotalRequestedMemOnHeap());
- allUserTopologies.add(td);
- }
- }
- return allUserTopologies;
- }
-
private static class SimulatedUserComparator implements Comparator<SimulatedUser> {
private final double cpuAvail;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/FIFOSchedulingPriorityStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/FIFOSchedulingPriorityStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/FIFOSchedulingPriorityStrategy.java
index 1c76b8d..98b3f4e 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/FIFOSchedulingPriorityStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/FIFOSchedulingPriorityStrategy.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource.strategies.priority;
@@ -29,6 +23,11 @@ import org.slf4j.LoggerFactory;
public class FIFOSchedulingPriorityStrategy extends DefaultSchedulingPriorityStrategy {
private static final Logger LOG = LoggerFactory.getLogger(FIFOSchedulingPriorityStrategy.class);
+ @Override
+ protected SimulatedUser getSimulatedUserFor(User u, ISchedulingState cluster) {
+ return new FIFOSimulatedUser(u, cluster);
+ }
+
protected static class FIFOSimulatedUser extends SimulatedUser {
public FIFOSimulatedUser(User other, ISchedulingState cluster) {
@@ -54,11 +53,6 @@ public class FIFOSchedulingPriorityStrategy extends DefaultSchedulingPriorityStr
}
}
- @Override
- protected SimulatedUser getSimulatedUserFor(User u, ISchedulingState cluster) {
- return new FIFOSimulatedUser(u, cluster);
- }
-
/**
* Comparator that sorts topologies by priority and then by submission time
* First sort by Topology Priority, if there is a tie for topology priority, topology uptime is used to sort
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/ISchedulingPriorityStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/ISchedulingPriorityStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/ISchedulingPriorityStrategy.java
index 77ce0d9..8bb50af 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/ISchedulingPriorityStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/priority/ISchedulingPriorityStrategy.java
@@ -1,26 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource.strategies.priority;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.scheduler.ISchedulingState;
import org.apache.storm.scheduler.TopologyDetails;
import org.apache.storm.scheduler.resource.User;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/BaseResourceAwareStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/BaseResourceAwareStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/BaseResourceAwareStrategy.java
index bd4d3ce..a474ccc 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/BaseResourceAwareStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/BaseResourceAwareStrategy.java
@@ -20,12 +20,10 @@ package org.apache.storm.scheduler.resource.strategies.scheduling;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
-
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -47,8 +45,8 @@ import org.slf4j.LoggerFactory;
public abstract class BaseResourceAwareStrategy implements IStrategy {
private static final Logger LOG = LoggerFactory.getLogger(BaseResourceAwareStrategy.class);
protected Cluster cluster;
- private Map<String, List<String>> networkTopography;
protected RAS_Nodes nodes;
+ private Map<String, List<String>> networkTopography;
@VisibleForTesting
void prepare(Cluster cluster) {
@@ -66,42 +64,43 @@ public abstract class BaseResourceAwareStrategy implements IStrategy {
/**
* Schedule executor exec from topology td.
*
- * @param exec the executor to schedule
- * @param td the topology executor exec is a part of
+ * @param exec the executor to schedule
+ * @param td the topology executor exec is a part of
* @param scheduledTasks executors that have been scheduled
*/
protected void scheduleExecutor(
- ExecutorDetails exec, TopologyDetails td, Collection<ExecutorDetails> scheduledTasks, List<ObjectResources> sortedNodes) {
+ ExecutorDetails exec, TopologyDetails td, Collection<ExecutorDetails> scheduledTasks, List<ObjectResources> sortedNodes) {
WorkerSlot targetSlot = findWorkerForExec(exec, td, sortedNodes);
if (targetSlot != null) {
RAS_Node targetNode = idToNode(targetSlot.getNodeId());
targetNode.assignSingleExecutor(targetSlot, exec, td);
scheduledTasks.add(exec);
LOG.debug(
- "TASK {} assigned to Node: {} avail [ mem: {} cpu: {} ] total [ mem: {} cpu: {} ] on "
- + "slot: {} on Rack: {}",
- exec,
- targetNode.getHostname(),
- targetNode.getAvailableMemoryResources(),
- targetNode.getAvailableCpuResources(),
- targetNode.getTotalMemoryResources(),
- targetNode.getTotalCpuResources(),
- targetSlot,
- nodeToRack(targetNode));
+ "TASK {} assigned to Node: {} avail [ mem: {} cpu: {} ] total [ mem: {} cpu: {} ] on "
+ + "slot: {} on Rack: {}",
+ exec,
+ targetNode.getHostname(),
+ targetNode.getAvailableMemoryResources(),
+ targetNode.getAvailableCpuResources(),
+ targetNode.getTotalMemoryResources(),
+ targetNode.getTotalCpuResources(),
+ targetSlot,
+ nodeToRack(targetNode));
} else {
LOG.error("Not Enough Resources to schedule Task {}", exec);
}
}
protected abstract TreeSet<ObjectResources> sortObjectResources(
- final AllResources allResources, ExecutorDetails exec, TopologyDetails topologyDetails,
- final ExistingScheduleFunc existingScheduleFunc
+ final AllResources allResources, ExecutorDetails exec, TopologyDetails topologyDetails,
+ final ExistingScheduleFunc existingScheduleFunc
);
+
/**
* Find a worker to schedule executor exec on.
*
* @param exec the executor to schedule
- * @param td the topology that the executor is a part of
+ * @param td the topology that the executor is a part of
* @return a worker to assign exec on. Returns null if a worker cannot be successfully found in cluster
*/
protected WorkerSlot findWorkerForExec(ExecutorDetails exec, TopologyDetails td, List<ObjectResources> sortedNodes) {
@@ -117,98 +116,24 @@ public abstract class BaseResourceAwareStrategy implements IStrategy {
}
/**
- * interface for calculating the number of existing executors scheduled on a object (rack or
- * node).
- */
- protected interface ExistingScheduleFunc {
- int getNumExistingSchedule(String objectId);
- }
-
- /**
- * a class to contain individual object resources as well as cumulative stats.
- */
- static class AllResources {
- List<ObjectResources> objectResources = new LinkedList<>();
- NormalizedResourceOffer availableResourcesOverall = new NormalizedResourceOffer();
- NormalizedResourceOffer totalResourcesOverall = new NormalizedResourceOffer();
- String identifier;
-
- public AllResources(String identifier) {
- this.identifier = identifier;
- }
-
- public AllResources(AllResources other) {
- this (null,
- new NormalizedResourceOffer(other.availableResourcesOverall),
- new NormalizedResourceOffer(other.totalResourcesOverall),
- other.identifier);
- List<ObjectResources> objectResourcesList = new ArrayList<>();
- for (ObjectResources objectResource : other.objectResources) {
- objectResourcesList.add(new ObjectResources(objectResource));
- }
- this.objectResources = objectResourcesList;
- }
-
- public AllResources(List<ObjectResources> objectResources, NormalizedResourceOffer availableResourcesOverall,
- NormalizedResourceOffer totalResourcesOverall, String identifier) {
- this.objectResources = objectResources;
- this.availableResourcesOverall = availableResourcesOverall;
- this.totalResourcesOverall = totalResourcesOverall;
- this.identifier = identifier;
- }
- }
-
- /**
- * class to keep track of resources on a rack or node.
- */
- static class ObjectResources {
- public final String id;
- public NormalizedResourceOffer availableResources = new NormalizedResourceOffer();
- public NormalizedResourceOffer totalResources = new NormalizedResourceOffer();
- public double effectiveResources = 0.0;
-
- public ObjectResources(String id) {
- this.id = id;
- }
-
- public ObjectResources(ObjectResources other) {
- this(other.id, other.availableResources, other.totalResources, other.effectiveResources);
- }
-
- public ObjectResources(String id, NormalizedResourceOffer availableResources, NormalizedResourceOffer totalResources,
- double effectiveResources) {
- this.id = id;
- this.availableResources = availableResources;
- this.totalResources = totalResources;
- this.effectiveResources = effectiveResources;
- }
-
- @Override
- public String toString() {
- return this.id;
- }
- }
-
- /**
* Nodes are sorted by two criteria.
*
* <p>1) the number executors of the topology that needs to be scheduled is already on the node in
- * descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a
- * topology on the same node as the existing executors of the topology.
+ * descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the same node as the
+ * existing executors of the topology.
*
* <p>2) the subordinate/subservient resource availability percentage of a node in descending
- * order We calculate the resource availability percentage by dividing the resource availability
- * that have exhausted or little of one of the resources mentioned above will be ranked after
- * on the node by the resource availability of the entire rack By doing this calculation, nodes
- * nodes that have more balanced resource availability. So we will be less likely to pick a node
- * that have a lot of one resource but a low amount of another.
+ * order We calculate the resource availability percentage by dividing the resource availability that have exhausted or little of one of
+ * the resources mentioned above will be ranked after on the node by the resource availability of the entire rack By doing this
+ * calculation, nodes nodes that have more balanced resource availability. So we will be less likely to pick a node that have a lot of
+ * one resource but a low amount of another.
*
* @param availNodes a list of all the nodes we want to sort
- * @param rackId the rack id availNodes are a part of
+ * @param rackId the rack id availNodes are a part of
* @return a sorted list of nodes.
*/
protected TreeSet<ObjectResources> sortNodes(
- List<RAS_Node> availNodes, ExecutorDetails exec, TopologyDetails topologyDetails, String rackId) {
+ List<RAS_Node> availNodes, ExecutorDetails exec, TopologyDetails topologyDetails, String rackId) {
AllResources allResources = new AllResources("RACK");
List<ObjectResources> nodes = allResources.objectResources;
@@ -264,7 +189,7 @@ public abstract class BaseResourceAwareStrategy implements IStrategy {
for (ObjectResources rack : sortedRacks) {
final String rackId = rack.id;
TreeSet<ObjectResources> sortedNodes = sortNodes(
- getAvailableNodesFromRack(rackId), exec, td, rackId);
+ getAvailableNodesFromRack(rackId), exec, td, rackId);
totallySortedNodes.addAll(sortedNodes);
}
//Now do some post processing to add make some nodes preferred over others.
@@ -304,14 +229,14 @@ public abstract class BaseResourceAwareStrategy implements IStrategy {
* Racks are sorted by two criteria.
*
* <p>1) the number executors of the topology that needs to be scheduled is already on the rack in descending order.
- * The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the same rack as the
- * existing executors of the topology.
+ * The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the same rack as the existing executors of the
+ * topology.
*
* <p>2) the subordinate/subservient resource availability percentage of a rack in descending order We calculate
- * the resource availability percentage by dividing the resource availability on the rack by the resource
- * availability of the entire cluster By doing this calculation, racks that have exhausted or little of one of
- * the resources mentioned above will be ranked after racks that have more balanced resource availability. So we
- * will be less likely to pick a rack that have a lot of one resource but a low amount of another.
+ * the resource availability percentage by dividing the resource availability on the rack by the resource availability of the entire
+ * cluster By doing this calculation, racks that have exhausted or little of one of the resources mentioned above will be ranked after
+ * racks that have more balanced resource availability. So we will be less likely to pick a rack that have a lot of one resource but a
+ * low amount of another.
*
* @return a sorted list of racks
*/
@@ -369,7 +294,6 @@ public abstract class BaseResourceAwareStrategy implements IStrategy {
});
}
-
/**
* Get the rack on which a node is a part of.
*
@@ -437,7 +361,7 @@ public abstract class BaseResourceAwareStrategy implements IStrategy {
/**
* Sort a component's neighbors by the number of connections it needs to make with this component.
*
- * @param thisComp the component that we need to sort its neighbors
+ * @param thisComp the component that we need to sort its neighbors
* @param componentMap all the components to sort
* @return a sorted set of components
*/
@@ -460,15 +384,14 @@ public abstract class BaseResourceAwareStrategy implements IStrategy {
}
/**
- * Order executors based on how many in and out connections it will potentially need to make, in descending order.
- * First order components by the number of in and out connections it will have. Then iterate through the sorted list of components.
- * For each component sort the neighbors of that component by how many connections it will have to make with that component.
- * Add an executor from this component and then from each neighboring component in sorted order.
- * Do this until there is nothing left to schedule.
+ * Order executors based on how many in and out connections it will potentially need to make, in descending order. First order
+ * components by the number of in and out connections it will have. Then iterate through the sorted list of components. For each
+ * component sort the neighbors of that component by how many connections it will have to make with that component. Add an executor from
+ * this component and then from each neighboring component in sorted order. Do this until there is nothing left to schedule.
*
- * @param td The topology the executors belong to
- * @param unassignedExecutors a collection of unassigned executors that need to be unassigned. Should only try to
- * assign executors from this list
+ * @param td The topology the executors belong to
+ * @param unassignedExecutors a collection of unassigned executors that need to be unassigned. Should only try to assign executors from
+ * this list
* @return a list of executors in sorted order
*/
protected List<ExecutorDetails> orderExecutors(
@@ -596,4 +519,76 @@ public abstract class BaseResourceAwareStrategy implements IStrategy {
}
return ret;
}
+
+ /**
+ * interface for calculating the number of existing executors scheduled on a object (rack or node).
+ */
+ protected interface ExistingScheduleFunc {
+ int getNumExistingSchedule(String objectId);
+ }
+
+ /**
+ * a class to contain individual object resources as well as cumulative stats.
+ */
+ static class AllResources {
+ List<ObjectResources> objectResources = new LinkedList<>();
+ NormalizedResourceOffer availableResourcesOverall = new NormalizedResourceOffer();
+ NormalizedResourceOffer totalResourcesOverall = new NormalizedResourceOffer();
+ String identifier;
+
+ public AllResources(String identifier) {
+ this.identifier = identifier;
+ }
+
+ public AllResources(AllResources other) {
+ this(null,
+ new NormalizedResourceOffer(other.availableResourcesOverall),
+ new NormalizedResourceOffer(other.totalResourcesOverall),
+ other.identifier);
+ List<ObjectResources> objectResourcesList = new ArrayList<>();
+ for (ObjectResources objectResource : other.objectResources) {
+ objectResourcesList.add(new ObjectResources(objectResource));
+ }
+ this.objectResources = objectResourcesList;
+ }
+
+ public AllResources(List<ObjectResources> objectResources, NormalizedResourceOffer availableResourcesOverall,
+ NormalizedResourceOffer totalResourcesOverall, String identifier) {
+ this.objectResources = objectResources;
+ this.availableResourcesOverall = availableResourcesOverall;
+ this.totalResourcesOverall = totalResourcesOverall;
+ this.identifier = identifier;
+ }
+ }
+
+ /**
+ * class to keep track of resources on a rack or node.
+ */
+ static class ObjectResources {
+ public final String id;
+ public NormalizedResourceOffer availableResources = new NormalizedResourceOffer();
+ public NormalizedResourceOffer totalResources = new NormalizedResourceOffer();
+ public double effectiveResources = 0.0;
+
+ public ObjectResources(String id) {
+ this.id = id;
+ }
+
+ public ObjectResources(ObjectResources other) {
+ this(other.id, other.availableResources, other.totalResources, other.effectiveResources);
+ }
+
+ public ObjectResources(String id, NormalizedResourceOffer availableResources, NormalizedResourceOffer totalResources,
+ double effectiveResources) {
+ this.id = id;
+ this.availableResources = availableResources;
+ this.totalResources = totalResources;
+ this.effectiveResources = effectiveResources;
+ }
+
+ @Override
+ public String toString() {
+ return this.id;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverStrategy.java
index f790050..12f89ca 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/ConstraintSolverStrategy.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.scheduler.resource.strategies.scheduling;
@@ -47,155 +41,201 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ConstraintSolverStrategy extends BaseResourceAwareStrategy {
+ //hard coded max number of states to search
+ public static final int MAX_STATE_SEARCH = 100_000;
+ public static final int DEFAULT_STATE_SEARCH = 10_000;
private static final Logger LOG = LoggerFactory.getLogger(ConstraintSolverStrategy.class);
+ private Map<String, RAS_Node> nodes;
+ private Map<ExecutorDetails, String> execToComp;
+ private Map<String, Set<ExecutorDetails>> compToExecs;
+ private List<String> favoredNodes;
+ private List<String> unFavoredNodes;
- protected static class SolverResult {
- private final int statesSearched;
- private final boolean success;
- private final long timeTakenMillis;
- private final int backtracked;
+ //constraints and spreads
+ private Map<String, Map<String, Integer>> constraintMatrix;
+ private HashSet<String> spreadComps = new HashSet<>();
- public SolverResult(SearcherState state, boolean success) {
- this.statesSearched = state.getStatesSearched();
- this.success = success;
- timeTakenMillis = Time.currentTimeMillis() - state.startTimeMillis;
- backtracked = state.numBacktrack;
+ static Map<String, Map<String, Integer>> getConstraintMap(TopologyDetails topo, Set<String> comps) {
+ Map<String, Map<String, Integer>> matrix = new HashMap<>();
+ for (String comp : comps) {
+ matrix.put(comp, new HashMap<>());
+ for (String comp2 : comps) {
+ matrix.get(comp).put(comp2, 0);
+ }
}
-
- public SchedulingResult asSchedulingResult() {
- if (success) {
- return SchedulingResult.success("Fully Scheduled by ConstraintSolverStrategy (" + statesSearched
- + " states traversed in " + timeTakenMillis + "ms, backtracked " + backtracked + " times)");
+ List<List<String>> constraints = (List<List<String>>) topo.getConf().get(Config.TOPOLOGY_RAS_CONSTRAINTS);
+ if (constraints != null) {
+ for (List<String> constraintPair : constraints) {
+ String comp1 = constraintPair.get(0);
+ String comp2 = constraintPair.get(1);
+ if (!matrix.containsKey(comp1)) {
+ LOG.warn("Comp: {} declared in constraints is not valid!", comp1);
+ continue;
+ }
+ if (!matrix.containsKey(comp2)) {
+ LOG.warn("Comp: {} declared in constraints is not valid!", comp2);
+ continue;
+ }
+ matrix.get(comp1).put(comp2, 1);
+ matrix.get(comp2).put(comp1, 1);
}
- return SchedulingResult.failure(SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES,
- "Cannot find scheduling that satisfies all constraints (" + statesSearched
- + " states traversed in " + timeTakenMillis + "ms, backtracked " + backtracked + " times)");
}
+ return matrix;
}
- protected static class SearcherState {
- // Metrics
- // How many states searched so far.
- private int statesSearched = 0;
- // Number of times we had to backtrack.
- private int numBacktrack = 0;
- final long startTimeMillis;
- private final long maxEndTimeMs;
-
- // Current state
- // The current executor we are trying to schedule
- private int execIndex = 0;
- // A map of the worker to the components in the worker to be able to enforce constraints.
- private final Map<WorkerSlot, Set<String>> workerCompAssignment;
- private final boolean[] okToRemoveFromWorker;
- // for the currently tested assignment a Map of the node to the components on it to be able to enforce constraints
- private final Map<RAS_Node, Set<String>> nodeCompAssignment;
- private final boolean[] okToRemoveFromNode;
-
- // Static State
- // The list of all executors (preferably sorted to make assignments simpler).
- private final List<ExecutorDetails> execs;
- //The maximum number of state to search before stopping.
- private final int maxStatesSearched;
- //The topology we are scheduling
- private final TopologyDetails td;
+ /**
+ * Determines if a scheduling is valid and all constraints are satisfied.
+ */
+ @VisibleForTesting
+ public static boolean validateSolution(Cluster cluster, TopologyDetails td) {
+ return checkSpreadSchedulingValid(cluster, td)
+ && checkConstraintsSatisfied(cluster, td)
+ && checkResourcesCorrect(cluster, td);
+ }
- private SearcherState(Map<WorkerSlot, Set<String>> workerCompAssignment, Map<RAS_Node, Set<String>> nodeCompAssignment,
- int maxStatesSearched, long maxTimeMs, List<ExecutorDetails> execs, TopologyDetails td) {
- assert !execs.isEmpty();
- assert execs != null;
+ /**
+ * Check if constraints are satisfied.
+ */
+ private static boolean checkConstraintsSatisfied(Cluster cluster, TopologyDetails topo) {
+ LOG.info("Checking constraints...");
+ Map<ExecutorDetails, WorkerSlot> result = cluster.getAssignmentById(topo.getId()).getExecutorToSlot();
+ Map<ExecutorDetails, String> execToComp = topo.getExecutorToComponent();
+ //get topology constraints
+ Map<String, Map<String, Integer>> constraintMatrix = getConstraintMap(topo, new HashSet<>(topo.getExecutorToComponent().values()));
- this.workerCompAssignment = workerCompAssignment;
- this.nodeCompAssignment = nodeCompAssignment;
- this.maxStatesSearched = maxStatesSearched;
- this.execs = execs;
- okToRemoveFromWorker = new boolean[execs.size()];
- okToRemoveFromNode = new boolean[execs.size()];
- this.td = td;
- startTimeMillis = Time.currentTimeMillis();
- if (maxTimeMs <= 0) {
- maxEndTimeMs = Long.MAX_VALUE;
- } else {
- maxEndTimeMs = startTimeMillis + maxTimeMs;
+ Map<WorkerSlot, Set<String>> workerCompMap = new HashMap<>();
+ result.forEach((exec, worker) -> {
+ String comp = execToComp.get(exec);
+ workerCompMap.computeIfAbsent(worker, (k) -> new HashSet<>()).add(comp);
+ });
+ for (Map.Entry<WorkerSlot, Set<String>> entry : workerCompMap.entrySet()) {
+ Set<String> comps = entry.getValue();
+ for (String comp1 : comps) {
+ for (String comp2 : comps) {
+ if (!comp1.equals(comp2) && constraintMatrix.get(comp1).get(comp2) != 0) {
+ LOG.error("Incorrect Scheduling: worker exclusion for Component {} and {} not satisfied on WorkerSlot: {}",
+ comp1, comp2, entry.getKey());
+ return false;
+ }
+ }
}
}
+ return true;
+ }
- public void incStatesSearched() {
- statesSearched++;
- if (LOG.isDebugEnabled() && statesSearched % 1_000 == 0) {
- LOG.debug("States Searched: {}", statesSearched);
- LOG.debug("backtrack: {}", numBacktrack);
+ private static Map<WorkerSlot, RAS_Node> workerToNodes(Cluster cluster) {
+ Map<WorkerSlot, RAS_Node> workerToNodes = new HashMap<>();
+ for (RAS_Node node : RAS_Nodes.getAllNodesFrom(cluster).values()) {
+ for (WorkerSlot s : node.getUsedSlots()) {
+ workerToNodes.put(s, node);
}
}
+ return workerToNodes;
+ }
- public int getStatesSearched() {
- return statesSearched;
- }
+ private static boolean checkSpreadSchedulingValid(Cluster cluster, TopologyDetails topo) {
+ LOG.info("Checking for a valid scheduling...");
+ Map<ExecutorDetails, WorkerSlot> result = cluster.getAssignmentById(topo.getId()).getExecutorToSlot();
+ Map<ExecutorDetails, String> execToComp = topo.getExecutorToComponent();
+ Map<WorkerSlot, HashSet<ExecutorDetails>> workerExecMap = new HashMap<>();
+ Map<WorkerSlot, HashSet<String>> workerCompMap = new HashMap<>();
+ Map<RAS_Node, HashSet<String>> nodeCompMap = new HashMap<>();
+ Map<WorkerSlot, RAS_Node> workerToNodes = workerToNodes(cluster);
+ boolean ret = true;
- public boolean areSearchLimitsExceeded() {
- return statesSearched > maxStatesSearched || Time.currentTimeMillis() > maxEndTimeMs;
- }
+ HashSet<String> spreadComps = getSpreadComps(topo);
+ for (Map.Entry<ExecutorDetails, WorkerSlot> entry : result.entrySet()) {
+ ExecutorDetails exec = entry.getKey();
+ WorkerSlot worker = entry.getValue();
+ RAS_Node node = workerToNodes.get(worker);
- public SearcherState nextExecutor() {
- execIndex++;
- if (execIndex >= execs.size()) {
- throw new IllegalStateException("Internal Error: exceeded the exec limit " + execIndex + " >= " + execs.size());
+ if (workerExecMap.computeIfAbsent(worker, (k) -> new HashSet<>()).contains(exec)) {
+ LOG.error("Incorrect Scheduling: Found duplicate in scheduling");
+ return false;
}
- return this;
+ workerExecMap.get(worker).add(exec);
+ String comp = execToComp.get(exec);
+ workerCompMap.computeIfAbsent(worker, (k) -> new HashSet<>()).add(comp);
+ if (spreadComps.contains(comp)) {
+ if (nodeCompMap.computeIfAbsent(node, (k) -> new HashSet<>()).contains(comp)) {
+ LOG.error("Incorrect Scheduling: Spread for Component: {} {} on node {} not satisfied {}",
+ comp, exec, node.getId(), nodeCompMap.get(node));
+ ret = false;
+ }
+ }
+ nodeCompMap.computeIfAbsent(node, (k) -> new HashSet<>()).add(comp);
}
+ return ret;
+ }
- public boolean areAllExecsScheduled() {
- return execIndex == execs.size() - 1;
+ /**
+ * Check if resource constraints satisfied.
+ */
+ private static boolean checkResourcesCorrect(Cluster cluster, TopologyDetails topo) {
+ LOG.info("Checking Resources...");
+ Map<ExecutorDetails, WorkerSlot> result = cluster.getAssignmentById(topo.getId()).getExecutorToSlot();
+ Map<RAS_Node, Collection<ExecutorDetails>> nodeToExecs = new HashMap<>();
+ Map<ExecutorDetails, WorkerSlot> mergedExecToWorker = new HashMap<>();
+ Map<String, RAS_Node> nodes = RAS_Nodes.getAllNodesFrom(cluster);
+ //merge with existing assignments
+ if (cluster.getAssignmentById(topo.getId()) != null
+ && cluster.getAssignmentById(topo.getId()).getExecutorToSlot() != null) {
+ mergedExecToWorker.putAll(cluster.getAssignmentById(topo.getId()).getExecutorToSlot());
}
+ mergedExecToWorker.putAll(result);
- public ExecutorDetails currentExec() {
- return execs.get(execIndex);
- }
+ for (Map.Entry<ExecutorDetails, WorkerSlot> entry : mergedExecToWorker.entrySet()) {
+ ExecutorDetails exec = entry.getKey();
+ WorkerSlot worker = entry.getValue();
+ RAS_Node node = nodes.get(worker.getNodeId());
- public void tryToSchedule(Map<ExecutorDetails, String> execToComp, RAS_Node node, WorkerSlot workerSlot) {
- ExecutorDetails exec = currentExec();
- String comp = execToComp.get(exec);
- LOG.trace("Trying assignment of {} {} to {}", exec, comp, workerSlot);
- //It is possible that this component is already scheduled on this node or worker. If so when we backtrack we cannot remove it
- okToRemoveFromWorker[execIndex] = workerCompAssignment.computeIfAbsent(workerSlot, (k) -> new HashSet<>()).add(comp);
- okToRemoveFromNode[execIndex] = nodeCompAssignment.computeIfAbsent(node, (k) -> new HashSet<>()).add(comp);
- node.assignSingleExecutor(workerSlot, exec, td);
+ if (node.getAvailableMemoryResources() < 0.0 && node.getAvailableCpuResources() < 0.0) {
+ LOG.error("Incorrect Scheduling: found node with negative available resources");
+ return false;
+ }
+ nodeToExecs.computeIfAbsent(node, (k) -> new HashSet<>()).add(exec);
}
- public void backtrack(Map<ExecutorDetails, String> execToComp, RAS_Node node, WorkerSlot workerSlot) {
- execIndex--;
- if (execIndex < 0) {
- throw new IllegalStateException("Internal Error: exec index became negative");
+ for (Map.Entry<RAS_Node, Collection<ExecutorDetails>> entry : nodeToExecs.entrySet()) {
+ RAS_Node node = entry.getKey();
+ Collection<ExecutorDetails> execs = entry.getValue();
+ double cpuUsed = 0.0;
+ double memoryUsed = 0.0;
+ for (ExecutorDetails exec : execs) {
+ cpuUsed += topo.getTotalCpuReqTask(exec);
+ memoryUsed += topo.getTotalMemReqTask(exec);
}
- numBacktrack++;
- ExecutorDetails exec = currentExec();
- String comp = execToComp.get(exec);
- LOG.trace("Backtracking {} {} from {}", exec, comp, workerSlot);
- if (okToRemoveFromWorker[execIndex]) {
- workerCompAssignment.get(workerSlot).remove(comp);
- okToRemoveFromWorker[execIndex] = false;
+ if (node.getAvailableCpuResources() != (node.getTotalCpuResources() - cpuUsed)) {
+ LOG.error("Incorrect Scheduling: node {} has consumed incorrect amount of cpu. Expected: {}"
+ + " Actual: {} Executors scheduled on node: {}",
+ node.getId(), (node.getTotalCpuResources() - cpuUsed), node.getAvailableCpuResources(), execs);
+ return false;
}
- if (okToRemoveFromNode[execIndex]) {
- nodeCompAssignment.get(node).remove(comp);
- okToRemoveFromNode[execIndex] = false;
+ if (node.getAvailableMemoryResources() != (node.getTotalMemoryResources() - memoryUsed)) {
+ LOG.error("Incorrect Scheduling: node {} has consumed incorrect amount of memory. Expected: {}"
+ + " Actual: {} Executors scheduled on node: {}",
+ node.getId(), (node.getTotalMemoryResources() - memoryUsed), node.getAvailableMemoryResources(), execs);
+ return false;
}
- node.freeSingleExecutor(exec, td);
}
+ return true;
}
- private Map<String, RAS_Node> nodes;
- private Map<ExecutorDetails, String> execToComp;
- private Map<String, Set<ExecutorDetails>> compToExecs;
- private List<String> favoredNodes;
- private List<String> unFavoredNodes;
-
- //constraints and spreads
- private Map<String, Map<String, Integer>> constraintMatrix;
- private HashSet<String> spreadComps = new HashSet<>();
-
- //hard coded max number of states to search
- public static final int MAX_STATE_SEARCH = 100_000;
- public static final int DEFAULT_STATE_SEARCH = 10_000;
+ private static HashSet<String> getSpreadComps(TopologyDetails topo) {
+ HashSet<String> retSet = new HashSet<>();
+ List<String> spread = (List<String>) topo.getConf().get(Config.TOPOLOGY_SPREAD_COMPONENTS);
+ if (spread != null) {
+ Set<String> comps = topo.getComponents().keySet();
+ for (String comp : spread) {
+ if (comps.contains(comp)) {
+ retSet.add(comp);
+ } else {
+ LOG.warn("Comp {} declared for spread not valid", comp);
+ }
+ }
+ }
+ return retSet;
+ }
@Override
public SchedulingResult schedule(Cluster cluster, TopologyDetails td) {
@@ -206,7 +246,8 @@ public class ConstraintSolverStrategy extends BaseResourceAwareStrategy {
Map<RAS_Node, Set<String>> nodeCompAssignment = new HashMap<>();
//set max number of states to search
final int maxStateSearch = Math.min(MAX_STATE_SEARCH,
- ObjectReader.getInt(td.getConf().get(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH), DEFAULT_STATE_SEARCH));
+ ObjectReader.getInt(td.getConf().get(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH),
+ DEFAULT_STATE_SEARCH));
final long maxTimeMs =
ObjectReader.getInt(td.getConf().get(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_TIME_SECS), -1).intValue() * 1000L;
@@ -228,8 +269,8 @@ public class ConstraintSolverStrategy extends BaseResourceAwareStrategy {
//get a sorted list of unassigned executors based on number of constraints
Set<ExecutorDetails> unassignedExecutors = new HashSet<>(cluster.getUnassignedExecutors(td));
List<ExecutorDetails> sortedExecs = getSortedExecs(spreadComps, constraintMatrix, compToExecs).stream()
- .filter(unassignedExecutors::contains)
- .collect(Collectors.toList());
+ .filter(unassignedExecutors::contains)
+ .collect(Collectors.toList());
//populate with existing assignments
SchedulerAssignment existingAssignment = cluster.getAssignmentById(td.getId());
@@ -258,18 +299,18 @@ public class ConstraintSolverStrategy extends BaseResourceAwareStrategy {
int numExecs = compToExecs.get(comp).size();
if (numExecs > nodes.size()) {
LOG.error("Unsatisfiable constraint: Component: {} marked as spread has {} executors which is larger "
- + "than number of nodes: {}", comp, numExecs, nodes.size());
+ + "than number of nodes: {}", comp, numExecs, nodes.size());
return false;
}
}
if (execToComp.size() >= MAX_STATE_SEARCH) {
LOG.error("Number of executors is greater than the maximum number of states allowed to be searched. "
- + "# of executors: {} Max states to search: {}", execToComp.size(), MAX_STATE_SEARCH);
+ + "# of executors: {} Max states to search: {}", execToComp.size(), MAX_STATE_SEARCH);
return false;
}
return true;
}
-
+
@Override
protected TreeSet<ObjectResources> sortObjectResources(
final AllResources allResources, ExecutorDetails exec, TopologyDetails topologyDetails,
@@ -289,233 +330,67 @@ public class ConstraintSolverStrategy extends BaseResourceAwareStrategy {
ExecutorDetails exec = state.currentExec();
List<ObjectResources> sortedNodes = sortAllNodes(state.td, exec, favoredNodes, unFavoredNodes);
- for (ObjectResources nodeResources: sortedNodes) {
+ for (ObjectResources nodeResources : sortedNodes) {
RAS_Node node = nodes.get(nodeResources.id);
for (WorkerSlot workerSlot : node.getSlotsAvailbleTo(state.td)) {
if (isExecAssignmentToWorkerValid(workerSlot, state)) {
- state.tryToSchedule(execToComp, node, workerSlot);
-
- if (state.areAllExecsScheduled()) {
- //Everything is scheduled correctly, so no need to search any more.
- return new SolverResult(state, true);
- }
-
- SolverResult results = backtrackSearch(state.nextExecutor());
- if (results.success) {
- //We found a good result we are done.
- return results;
- }
-
- if (state.areSearchLimitsExceeded()) {
- //No need to search more it is not going to help.
- return new SolverResult(state, false);
- }
-
- //backtracking (If we ever get here there really isn't a lot of hope that we will find a scheduling)
- state.backtrack(execToComp, node, workerSlot);
- }
- }
- }
- //Tried all of the slots and none of them worked.
- return new SolverResult(state, false);
- }
-
- /**
- * Check if any constraints are violated if exec is scheduled on worker.
- * @return true if scheduling exec on worker does not violate any constraints, returns false if it does
- */
- public boolean isExecAssignmentToWorkerValid(WorkerSlot worker, SearcherState state) {
- final ExecutorDetails exec = state.currentExec();
- //check resources
- RAS_Node node = nodes.get(worker.getNodeId());
- if (!node.wouldFit(worker, exec, state.td)) {
- LOG.trace("{} would not fit in resources available on {}", exec, worker);
- return false;
- }
-
- //check if exec can be on worker based on user defined component exclusions
- String execComp = execToComp.get(exec);
- Set<String> components = state.workerCompAssignment.get(worker);
- if (components != null) {
- Map<String, Integer> subMatrix = constraintMatrix.get(execComp);
- for (String comp : components) {
- if (subMatrix.get(comp) != 0) {
- LOG.trace("{} found {} constraint violation {} on {}", exec, execComp, comp, worker);
- return false;
- }
- }
- }
-
- //check if exec satisfy spread
- if (spreadComps.contains(execComp)) {
- if (state.nodeCompAssignment.computeIfAbsent(node, (k) -> new HashSet<>()).contains(execComp)) {
- LOG.trace("{} Found spread violation {} on node {}", exec, execComp, node.getId());
- return false;
- }
- }
- return true;
- }
-
- static Map<String, Map<String, Integer>> getConstraintMap(TopologyDetails topo, Set<String> comps) {
- Map<String, Map<String, Integer>> matrix = new HashMap<>();
- for (String comp : comps) {
- matrix.put(comp, new HashMap<>());
- for (String comp2 : comps) {
- matrix.get(comp).put(comp2, 0);
- }
- }
- List<List<String>> constraints = (List<List<String>>) topo.getConf().get(Config.TOPOLOGY_RAS_CONSTRAINTS);
- if (constraints != null) {
- for (List<String> constraintPair : constraints) {
- String comp1 = constraintPair.get(0);
- String comp2 = constraintPair.get(1);
- if (!matrix.containsKey(comp1)) {
- LOG.warn("Comp: {} declared in constraints is not valid!", comp1);
- continue;
- }
- if (!matrix.containsKey(comp2)) {
- LOG.warn("Comp: {} declared in constraints is not valid!", comp2);
- continue;
- }
- matrix.get(comp1).put(comp2, 1);
- matrix.get(comp2).put(comp1, 1);
- }
- }
- return matrix;
- }
-
- /**
- * Determines if a scheduling is valid and all constraints are satisfied.
- */
- @VisibleForTesting
- public static boolean validateSolution(Cluster cluster, TopologyDetails td) {
- return checkSpreadSchedulingValid(cluster, td)
- && checkConstraintsSatisfied(cluster, td)
- && checkResourcesCorrect(cluster, td);
- }
-
- /**
- * Check if constraints are satisfied.
- */
- private static boolean checkConstraintsSatisfied(Cluster cluster, TopologyDetails topo) {
- LOG.info("Checking constraints...");
- Map<ExecutorDetails, WorkerSlot> result = cluster.getAssignmentById(topo.getId()).getExecutorToSlot();
- Map<ExecutorDetails, String> execToComp = topo.getExecutorToComponent();
- //get topology constraints
- Map<String, Map<String, Integer>> constraintMatrix = getConstraintMap(topo, new HashSet<>(topo.getExecutorToComponent().values()));
-
- Map<WorkerSlot, Set<String>> workerCompMap = new HashMap<>();
- result.forEach((exec, worker) -> {
- String comp = execToComp.get(exec);
- workerCompMap.computeIfAbsent(worker, (k) -> new HashSet<>()).add(comp);
- });
- for (Map.Entry<WorkerSlot, Set<String>> entry : workerCompMap.entrySet()) {
- Set<String> comps = entry.getValue();
- for (String comp1 : comps) {
- for (String comp2: comps) {
- if (!comp1.equals(comp2) && constraintMatrix.get(comp1).get(comp2) != 0) {
- LOG.error("Incorrect Scheduling: worker exclusion for Component {} and {} not satisfied on WorkerSlot: {}",
- comp1, comp2, entry.getKey());
- return false;
- }
- }
- }
- }
- return true;
- }
+ state.tryToSchedule(execToComp, node, workerSlot);
- private static Map<WorkerSlot, RAS_Node> workerToNodes(Cluster cluster) {
- Map<WorkerSlot, RAS_Node> workerToNodes = new HashMap<>();
- for (RAS_Node node: RAS_Nodes.getAllNodesFrom(cluster).values()) {
- for (WorkerSlot s : node.getUsedSlots()) {
- workerToNodes.put(s, node);
- }
- }
- return workerToNodes;
- }
+ if (state.areAllExecsScheduled()) {
+ //Everything is scheduled correctly, so no need to search any more.
+ return new SolverResult(state, true);
+ }
- private static boolean checkSpreadSchedulingValid(Cluster cluster, TopologyDetails topo) {
- LOG.info("Checking for a valid scheduling...");
- Map<ExecutorDetails, WorkerSlot> result = cluster.getAssignmentById(topo.getId()).getExecutorToSlot();
- Map<ExecutorDetails, String> execToComp = topo.getExecutorToComponent();
- Map<WorkerSlot, HashSet<ExecutorDetails>> workerExecMap = new HashMap<>();
- Map<WorkerSlot, HashSet<String>> workerCompMap = new HashMap<>();
- Map<RAS_Node, HashSet<String>> nodeCompMap = new HashMap<>();
- Map<WorkerSlot, RAS_Node> workerToNodes = workerToNodes(cluster);
- boolean ret = true;
+ SolverResult results = backtrackSearch(state.nextExecutor());
+ if (results.success) {
+ //We found a good result we are done.
+ return results;
+ }
- HashSet<String> spreadComps = getSpreadComps(topo);
- for (Map.Entry<ExecutorDetails, WorkerSlot> entry : result.entrySet()) {
- ExecutorDetails exec = entry.getKey();
- WorkerSlot worker = entry.getValue();
- RAS_Node node = workerToNodes.get(worker);
+ if (state.areSearchLimitsExceeded()) {
+ //No need to search more it is not going to help.
+ return new SolverResult(state, false);
+ }
- if (workerExecMap.computeIfAbsent(worker, (k) -> new HashSet<>()).contains(exec)) {
- LOG.error("Incorrect Scheduling: Found duplicate in scheduling");
- return false;
- }
- workerExecMap.get(worker).add(exec);
- String comp = execToComp.get(exec);
- workerCompMap.computeIfAbsent(worker, (k) -> new HashSet<>()).add(comp);
- if (spreadComps.contains(comp)) {
- if (nodeCompMap.computeIfAbsent(node, (k) -> new HashSet<>()).contains(comp)) {
- LOG.error("Incorrect Scheduling: Spread for Component: {} {} on node {} not satisfied {}",
- comp, exec, node.getId(), nodeCompMap.get(node));
- ret = false;
+ //backtracking (If we ever get here there really isn't a lot of hope that we will find a scheduling)
+ state.backtrack(execToComp, node, workerSlot);
}
}
- nodeCompMap.computeIfAbsent(node, (k) -> new HashSet<>()).add(comp);
}
- return ret;
+ //Tried all of the slots and none of them worked.
+ return new SolverResult(state, false);
}
/**
- * Check if resource constraints satisfied.
+ * Check if any constraints are violated if exec is scheduled on worker.
+ * @return true if scheduling exec on worker does not violate any constraints, returns false if it does
*/
- private static boolean checkResourcesCorrect(Cluster cluster, TopologyDetails topo) {
- LOG.info("Checking Resources...");
- Map<ExecutorDetails, WorkerSlot> result = cluster.getAssignmentById(topo.getId()).getExecutorToSlot();
- Map<RAS_Node, Collection<ExecutorDetails>> nodeToExecs = new HashMap<>();
- Map<ExecutorDetails, WorkerSlot> mergedExecToWorker = new HashMap<>();
- Map<String, RAS_Node> nodes = RAS_Nodes.getAllNodesFrom(cluster);
- //merge with existing assignments
- if (cluster.getAssignmentById(topo.getId()) != null
- && cluster.getAssignmentById(topo.getId()).getExecutorToSlot() != null) {
- mergedExecToWorker.putAll(cluster.getAssignmentById(topo.getId()).getExecutorToSlot());
+ public boolean isExecAssignmentToWorkerValid(WorkerSlot worker, SearcherState state) {
+ final ExecutorDetails exec = state.currentExec();
+ //check resources
+ RAS_Node node = nodes.get(worker.getNodeId());
+ if (!node.wouldFit(worker, exec, state.td)) {
+ LOG.trace("{} would not fit in resources available on {}", exec, worker);
+ return false;
}
- mergedExecToWorker.putAll(result);
-
- for (Map.Entry<ExecutorDetails, WorkerSlot> entry : mergedExecToWorker.entrySet()) {
- ExecutorDetails exec = entry.getKey();
- WorkerSlot worker = entry.getValue();
- RAS_Node node = nodes.get(worker.getNodeId());
- if (node.getAvailableMemoryResources() < 0.0 && node.getAvailableCpuResources() < 0.0) {
- LOG.error("Incorrect Scheduling: found node with negative available resources");
- return false;
+ //check if exec can be on worker based on user defined component exclusions
+ String execComp = execToComp.get(exec);
+ Set<String> components = state.workerCompAssignment.get(worker);
+ if (components != null) {
+ Map<String, Integer> subMatrix = constraintMatrix.get(execComp);
+ for (String comp : components) {
+ if (subMatrix.get(comp) != 0) {
+ LOG.trace("{} found {} constraint violation {} on {}", exec, execComp, comp, worker);
+ return false;
+ }
}
- nodeToExecs.computeIfAbsent(node, (k) -> new HashSet<>()).add(exec);
}
- for (Map.Entry<RAS_Node, Collection<ExecutorDetails>> entry : nodeToExecs.entrySet()) {
- RAS_Node node = entry.getKey();
- Collection<ExecutorDetails> execs = entry.getValue();
- double cpuUsed = 0.0;
- double memoryUsed = 0.0;
- for (ExecutorDetails exec : execs) {
- cpuUsed += topo.getTotalCpuReqTask(exec);
- memoryUsed += topo.getTotalMemReqTask(exec);
- }
- if (node.getAvailableCpuResources() != (node.getTotalCpuResources() - cpuUsed)) {
- LOG.error("Incorrect Scheduling: node {} has consumed incorrect amount of cpu. Expected: {}"
- + " Actual: {} Executors scheduled on node: {}",
- node.getId(), (node.getTotalCpuResources() - cpuUsed), node.getAvailableCpuResources(), execs);
- return false;
- }
- if (node.getAvailableMemoryResources() != (node.getTotalMemoryResources() - memoryUsed)) {
- LOG.error("Incorrect Scheduling: node {} has consumed incorrect amount of memory. Expected: {}"
- + " Actual: {} Executors scheduled on node: {}",
- node.getId(), (node.getTotalMemoryResources() - memoryUsed), node.getAvailableMemoryResources(), execs);
+ //check if exec satisfy spread
+ if (spreadComps.contains(execComp)) {
+ if (state.nodeCompAssignment.computeIfAbsent(node, (k) -> new HashSet<>()).contains(execComp)) {
+ LOG.trace("{} Found spread violation {} on node {}", exec, execComp, node.getId());
return false;
}
}
@@ -551,22 +426,6 @@ public class ConstraintSolverStrategy extends BaseResourceAwareStrategy {
return retList;
}
- private static HashSet<String> getSpreadComps(TopologyDetails topo) {
- HashSet<String> retSet = new HashSet<>();
- List<String> spread = (List<String>) topo.getConf().get(Config.TOPOLOGY_SPREAD_COMPONENTS);
- if (spread != null) {
- Set<String> comps = topo.getComponents().keySet();
- for (String comp : spread) {
- if (comps.contains(comp)) {
- retSet.add(comp);
- } else {
- LOG.warn("Comp {} declared for spread not valid", comp);
- }
- }
- }
- return retSet;
- }
-
/**
* Used to sort a Map by the values.
*/
@@ -584,4 +443,136 @@ public class ConstraintSolverStrategy extends BaseResourceAwareStrategy {
sortedByValues.putAll(map);
return sortedByValues;
}
+
+ protected static class SolverResult {
+ private final int statesSearched;
+ private final boolean success;
+ private final long timeTakenMillis;
+ private final int backtracked;
+
+ public SolverResult(SearcherState state, boolean success) {
+ this.statesSearched = state.getStatesSearched();
+ this.success = success;
+ timeTakenMillis = Time.currentTimeMillis() - state.startTimeMillis;
+ backtracked = state.numBacktrack;
+ }
+
+ public SchedulingResult asSchedulingResult() {
+ if (success) {
+ return SchedulingResult.success("Fully Scheduled by ConstraintSolverStrategy (" + statesSearched
+ + " states traversed in " + timeTakenMillis + "ms, backtracked " + backtracked + " times)");
+ }
+ return SchedulingResult.failure(SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES,
+ "Cannot find scheduling that satisfies all constraints (" + statesSearched
+ + " states traversed in " + timeTakenMillis + "ms, backtracked " + backtracked + " times)");
+ }
+ }
+
+ protected static class SearcherState {
+ final long startTimeMillis;
+ private final long maxEndTimeMs;
+ // A map of the worker to the components in the worker to be able to enforce constraints.
+ private final Map<WorkerSlot, Set<String>> workerCompAssignment;
+ private final boolean[] okToRemoveFromWorker;
+ // for the currently tested assignment a Map of the node to the components on it to be able to enforce constraints
+ private final Map<RAS_Node, Set<String>> nodeCompAssignment;
+ private final boolean[] okToRemoveFromNode;
+ // Static State
+ // The list of all executors (preferably sorted to make assignments simpler).
+ private final List<ExecutorDetails> execs;
+ //The maximum number of state to search before stopping.
+ private final int maxStatesSearched;
+ //The topology we are scheduling
+ private final TopologyDetails td;
+ // Metrics
+ // How many states searched so far.
+ private int statesSearched = 0;
+ // Number of times we had to backtrack.
+ private int numBacktrack = 0;
+ // Current state
+ // The current executor we are trying to schedule
+ private int execIndex = 0;
+
+ private SearcherState(Map<WorkerSlot, Set<String>> workerCompAssignment, Map<RAS_Node, Set<String>> nodeCompAssignment,
+ int maxStatesSearched, long maxTimeMs, List<ExecutorDetails> execs, TopologyDetails td) {
+ assert !execs.isEmpty();
+ assert execs != null;
+
+ this.workerCompAssignment = workerCompAssignment;
+ this.nodeCompAssignment = nodeCompAssignment;
+ this.maxStatesSearched = maxStatesSearched;
+ this.execs = execs;
+ okToRemoveFromWorker = new boolean[execs.size()];
+ okToRemoveFromNode = new boolean[execs.size()];
+ this.td = td;
+ startTimeMillis = Time.currentTimeMillis();
+ if (maxTimeMs <= 0) {
+ maxEndTimeMs = Long.MAX_VALUE;
+ } else {
+ maxEndTimeMs = startTimeMillis + maxTimeMs;
+ }
+ }
+
+ public void incStatesSearched() {
+ statesSearched++;
+ if (LOG.isDebugEnabled() && statesSearched % 1_000 == 0) {
+ LOG.debug("States Searched: {}", statesSearched);
+ LOG.debug("backtrack: {}", numBacktrack);
+ }
+ }
+
+ public int getStatesSearched() {
+ return statesSearched;
+ }
+
+ public boolean areSearchLimitsExceeded() {
+ return statesSearched > maxStatesSearched || Time.currentTimeMillis() > maxEndTimeMs;
+ }
+
+ public SearcherState nextExecutor() {
+ execIndex++;
+ if (execIndex >= execs.size()) {
+ throw new IllegalStateException("Internal Error: exceeded the exec limit " + execIndex + " >= " + execs.size());
+ }
+ return this;
+ }
+
+ public boolean areAllExecsScheduled() {
+ return execIndex == execs.size() - 1;
+ }
+
+ public ExecutorDetails currentExec() {
+ return execs.get(execIndex);
+ }
+
+ public void tryToSchedule(Map<ExecutorDetails, String> execToComp, RAS_Node node, WorkerSlot workerSlot) {
+ ExecutorDetails exec = currentExec();
+ String comp = execToComp.get(exec);
+ LOG.trace("Trying assignment of {} {} to {}", exec, comp, workerSlot);
+ //It is possible that this component is already scheduled on this node or worker. If so when we backtrack we cannot remove it
+ okToRemoveFromWorker[execIndex] = workerCompAssignment.computeIfAbsent(workerSlot, (k) -> new HashSet<>()).add(comp);
+ okToRemoveFromNode[execIndex] = nodeCompAssignment.computeIfAbsent(node, (k) -> new HashSet<>()).add(comp);
+ node.assignSingleExecutor(workerSlot, exec, td);
+ }
+
+ public void backtrack(Map<ExecutorDetails, String> execToComp, RAS_Node node, WorkerSlot workerSlot) {
+ execIndex--;
+ if (execIndex < 0) {
+ throw new IllegalStateException("Internal Error: exec index became negative");
+ }
+ numBacktrack++;
+ ExecutorDetails exec = currentExec();
+ String comp = execToComp.get(exec);
+ LOG.trace("Backtracking {} {} from {}", exec, comp, workerSlot);
+ if (okToRemoveFromWorker[execIndex]) {
+ workerCompAssignment.get(workerSlot).remove(comp);
+ okToRemoveFromWorker[execIndex] = false;
+ }
+ if (okToRemoveFromNode[execIndex]) {
+ nodeCompAssignment.get(node).remove(comp);
+ okToRemoveFromNode[execIndex] = false;
+ }
+ node.freeSingleExecutor(exec, td);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/DefaultResourceAwareStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/DefaultResourceAwareStrategy.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/DefaultResourceAwareStrategy.java
index efa3ecf..826c24d 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/DefaultResourceAwareStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/strategies/scheduling/DefaultResourceAwareStrategy.java
@@ -43,10 +43,10 @@ public class DefaultResourceAwareStrategy extends BaseResourceAwareStrategy impl
if (nodes.getNodes().size() <= 0) {
LOG.warn("No available nodes to schedule tasks on!");
return SchedulingResult.failure(
- SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES, "No available nodes to schedule tasks on!");
+ SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES, "No available nodes to schedule tasks on!");
}
Collection<ExecutorDetails> unassignedExecutors =
- new HashSet<>(this.cluster.getUnassignedExecutors(td));
+ new HashSet<>(this.cluster.getUnassignedExecutors(td));
LOG.debug("ExecutorsNeedScheduling: {}", unassignedExecutors);
Collection<ExecutorDetails> scheduledTasks = new ArrayList<>();
List<Component> spouts = this.getSpouts(td);
@@ -54,7 +54,7 @@ public class DefaultResourceAwareStrategy extends BaseResourceAwareStrategy impl
if (spouts.size() == 0) {
LOG.error("Cannot find a Spout!");
return SchedulingResult.failure(
- SchedulingStatus.FAIL_INVALID_TOPOLOGY, "Cannot find a Spout!");
+ SchedulingStatus.FAIL_INVALID_TOPOLOGY, "Cannot find a Spout!");
}
//order executors to be scheduled
@@ -66,10 +66,10 @@ public class DefaultResourceAwareStrategy extends BaseResourceAwareStrategy impl
for (ExecutorDetails exec : orderedExecutors) {
LOG.debug(
- "Attempting to schedule: {} of component {}[ REQ {} ]",
- exec,
- td.getExecutorToComponent().get(exec),
- td.getTaskResourceReqList(exec));
+ "Attempting to schedule: {} of component {}[ REQ {} ]",
+ exec,
+ td.getExecutorToComponent().get(exec),
+ td.getTaskResourceReqList(exec));
scheduleExecutor(exec, td, scheduledTasks, sortedNodes);
}
@@ -85,12 +85,12 @@ public class DefaultResourceAwareStrategy extends BaseResourceAwareStrategy impl
if (executorsNotScheduled.size() > 0) {
LOG.error("Not all executors successfully scheduled: {}", executorsNotScheduled);
result =
- SchedulingResult.failure(
- SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES,
- (td.getExecutors().size() - unassignedExecutors.size())
- + "/"
- + td.getExecutors().size()
- + " executors scheduled");
+ SchedulingResult.failure(
+ SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES,
+ (td.getExecutors().size() - unassignedExecutors.size())
+ + "/"
+ + td.getExecutors().size()
+ + " executors scheduled");
} else {
LOG.debug("All resources successfully scheduled!");
result = SchedulingResult.success("Fully Scheduled by " + this.getClass().getSimpleName());
@@ -99,64 +99,61 @@ public class DefaultResourceAwareStrategy extends BaseResourceAwareStrategy impl
}
/**
- * Sort objects by the following two criteria. 1) the number executors of the topology that needs
- * to be scheduled is already on the object (node or rack) in descending order. The reasoning to
- * sort based on criterion 1 is so we schedule the rest of a topology on the same object (node or
- * rack) as the existing executors of the topology. 2) the subordinate/subservient resource
- * availability percentage of a rack in descending order We calculate the resource availability
- * percentage by dividing the resource availability of the object (node or rack) by the resource
- * availability of the entire rack or cluster depending on if object references a node or a rack.
- * By doing this calculation, objects (node or rack) that have exhausted or little of one of the
- * resources mentioned above will be ranked after racks that have more balanced resource
- * availability. So we will be less likely to pick a rack that have a lot of one resource but a
- * low amount of another.
+ * Sort objects by the following two criteria. 1) the number executors of the topology that needs to be scheduled is already on the
+ * object (node or rack) in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on the
+ * same object (node or rack) as the existing executors of the topology. 2) the subordinate/subservient resource availability percentage
+ * of a rack in descending order We calculate the resource availability percentage by dividing the resource availability of the object
+ * (node or rack) by the resource availability of the entire rack or cluster depending on if object references a node or a rack. By
+ * doing this calculation, objects (node or rack) that have exhausted or little of one of the resources mentioned above will be ranked
+ * after racks that have more balanced resource availability. So we will be less likely to pick a rack that have a lot of one resource
+ * but a low amount of another.
*
- * @param allResources contains all individual ObjectResources as well as cumulative stats
+ * @param allResources contains all individual ObjectResources as well as cumulative stats
* @param existingScheduleFunc a function to get existing executors already scheduled on this object
* @return a sorted list of ObjectResources
*/
@Override
protected TreeSet<ObjectResources> sortObjectResources(
- final AllResources allResources, ExecutorDetails exec, TopologyDetails topologyDetails,
- final ExistingScheduleFunc existingScheduleFunc) {
-
+ final AllResources allResources, ExecutorDetails exec, TopologyDetails topologyDetails,
+ final ExistingScheduleFunc existingScheduleFunc) {
+
for (ObjectResources objectResources : allResources.objectResources) {
objectResources.effectiveResources =
allResources.availableResourcesOverall.calculateMinPercentageUsedBy(objectResources.availableResources);
if (LOG.isTraceEnabled()) {
LOG.trace("Effective resources for {} is {}, and numExistingSchedule is {}",
- objectResources.id, objectResources.effectiveResources,
- existingScheduleFunc.getNumExistingSchedule(objectResources.id));
+ objectResources.id, objectResources.effectiveResources,
+ existingScheduleFunc.getNumExistingSchedule(objectResources.id));
}
}
TreeSet<ObjectResources> sortedObjectResources =
- new TreeSet<>((o1, o2) -> {
- int execsScheduled1 = existingScheduleFunc.getNumExistingSchedule(o1.id);
- int execsScheduled2 = existingScheduleFunc.getNumExistingSchedule(o2.id);
- if (execsScheduled1 > execsScheduled2) {
+ new TreeSet<>((o1, o2) -> {
+ int execsScheduled1 = existingScheduleFunc.getNumExistingSchedule(o1.id);
+ int execsScheduled2 = existingScheduleFunc.getNumExistingSchedule(o2.id);
+ if (execsScheduled1 > execsScheduled2) {
+ return -1;
+ } else if (execsScheduled1 < execsScheduled2) {
+ return 1;
+ } else {
+ if (o1.effectiveResources > o2.effectiveResources) {
return -1;
- } else if (execsScheduled1 < execsScheduled2) {
+ } else if (o1.effectiveResources < o2.effectiveResources) {
return 1;
} else {
- if (o1.effectiveResources > o2.effectiveResources) {
+ double o1Avg = allResources.availableResourcesOverall.calculateAveragePercentageUsedBy(o1.availableResources);
+ double o2Avg = allResources.availableResourcesOverall.calculateAveragePercentageUsedBy(o2.availableResources);
+
+ if (o1Avg > o2Avg) {
return -1;
- } else if (o1.effectiveResources < o2.effectiveResources) {
+ } else if (o1Avg < o2Avg) {
return 1;
} else {
- double o1Avg = allResources.availableResourcesOverall.calculateAveragePercentageUsedBy(o1.availableResources);
- double o2Avg = allResources.availableResourcesOverall.calculateAveragePercentageUsedBy(o2.availableResources);
-
- if (o1Avg > o2Avg) {
- return -1;
- } else if (o1Avg < o2Avg) {
- return 1;
- } else {
- return o1.id.compareTo(o2.id);
- }
+ return o1.id.compareTo(o2.id);
}
}
- });
+ }
+ });
sortedObjectResources.addAll(allResources.objectResources);
LOG.debug("Sorted Object Resources: {}", sortedObjectResources);
return sortedObjectResources;
[22/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-core
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-core
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fc1cf09b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fc1cf09b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fc1cf09b
Branch: refs/heads/master
Commit: fc1cf09b05c86dc5d1802049e7b650ceb52e54d4
Parents: f1c0bcb
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 21:12:29 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 00:04:03 2018 -0400
----------------------------------------------------------------------
storm-core/pom.xml | 2 +-
.../jvm/org/apache/storm/command/Activate.java | 32 +-
.../org/apache/storm/command/AdminCommands.java | 39 +--
.../apache/storm/command/BasicDrpcClient.java | 30 +-
.../jvm/org/apache/storm/command/Blobstore.java | 85 +++---
.../src/jvm/org/apache/storm/command/CLI.java | 303 +++++++++----------
.../org/apache/storm/command/ConfigValue.java | 26 +-
.../org/apache/storm/command/Deactivate.java | 32 +-
.../org/apache/storm/command/DevZookeeper.java | 20 +-
.../jvm/org/apache/storm/command/GetErrors.java | 28 +-
.../org/apache/storm/command/HealthCheck.java | 20 +-
.../org/apache/storm/command/Heartbeats.java | 23 +-
.../org/apache/storm/command/KillTopology.java | 39 +--
.../org/apache/storm/command/KillWorkers.java | 24 +-
.../apache/storm/command/ListTopologies.java | 55 ++--
.../jvm/org/apache/storm/command/Monitor.java | 31 +-
.../jvm/org/apache/storm/command/Rebalance.java | 39 +--
.../org/apache/storm/command/SetLogLevel.java | 32 +-
.../apache/storm/command/UploadCredentials.java | 40 ++-
.../org/apache/storm/planner/CompoundSpout.java | 21 +-
.../org/apache/storm/planner/CompoundTask.java | 21 +-
.../org/apache/storm/planner/TaskBundle.java | 25 +-
.../apache/storm/testing/MockLeaderElector.java | 23 +-
.../staticmocking/MockedClientZookeeper.java | 1 +
.../testing/staticmocking/MockedCluster.java | 20 +-
.../staticmocking/MockedSupervisorUtils.java | 18 +-
.../testing/staticmocking/MockedZookeeper.java | 18 +-
.../apache/storm/ui/FilterConfiguration.java | 19 +-
.../jvm/org/apache/storm/ui/IConfigurator.java | 19 +-
.../storm/ui/InvalidRequestException.java | 1 +
.../src/jvm/org/apache/storm/ui/UIHelpers.java | 70 ++---
.../apache/storm/utils/ClojureTimerTask.java | 25 +-
.../src/jvm/org/apache/storm/utils/Monitor.java | 185 +++++------
.../apache/storm/utils/TopologySpoutLag.java | 75 +++--
.../test/jvm/org/apache/storm/MockAutoCred.java | 25 +-
.../org/apache/storm/command/RebalanceTest.java | 21 +-
.../apache/storm/command/SetLogLevelTest.java | 21 +-
.../jvm/org/apache/storm/command/TestCLI.java | 48 ++-
.../apache/storm/metric/FakeMetricConsumer.java | 41 ++-
.../nimbus/InMemoryTopologyActionNotifier.java | 27 +-
.../storm/serialization/SerializationTest.java | 10 +-
.../org/apache/storm/stats/TestStatsUtil.java | 64 ++--
.../staticmocking/ConfigUtilsInstaller.java | 3 +-
.../staticmocking/ReflectionUtilsInstaller.java | 3 +-
.../ServerConfigUtilsInstaller.java | 3 +-
.../utils/staticmocking/UtilsInstaller.java | 3 +-
.../storm/utils/staticmocking/package-info.java | 66 +++-
47 files changed, 807 insertions(+), 969 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 88074b8..26109f9 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -449,7 +449,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>254</maxAllowedViolations>
+ <maxAllowedViolations>73</maxAllowedViolations>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/Activate.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/Activate.java b/storm-core/src/jvm/org/apache/storm/command/Activate.java
index 45d4a99..73ad4f5 100644
--- a/storm-core/src/jvm/org/apache/storm/command/Activate.java
+++ b/storm-core/src/jvm/org/apache/storm/command/Activate.java
@@ -1,40 +1,34 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.command;
import org.apache.storm.generated.Nimbus;
import org.apache.storm.utils.NimbusClient;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Activate {
private static final Logger LOG = LoggerFactory.getLogger(Activate.class);
- public static void main(String [] args) throws Exception {
+ public static void main(String[] args) throws Exception {
final String name = args[0];
NimbusClient.withConfiguredClient(new NimbusClient.WithNimbus() {
- @Override
- public void run(Nimbus.Iface nimbus) throws Exception {
- nimbus.activate(name);
- LOG.info("Activated topology: {}", name);
- }
+ @Override
+ public void run(Nimbus.Iface nimbus) throws Exception {
+ nimbus.activate(name);
+ LOG.info("Activated topology: {}", name);
+ }
});
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/AdminCommands.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/AdminCommands.java b/storm-core/src/jvm/org/apache/storm/command/AdminCommands.java
index 8d66a61..7ae7dbc 100644
--- a/storm-core/src/jvm/org/apache/storm/command/AdminCommands.java
+++ b/storm-core/src/jvm/org/apache/storm/command/AdminCommands.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.command;
@@ -21,7 +15,6 @@ package org.apache.storm.command;
import com.google.common.collect.Sets;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.storm.blobstore.BlobStore;
@@ -45,7 +38,7 @@ public class AdminCommands {
private static IStormClusterState stormClusterState;
private static Map<String, Object> conf;
- public static void main(String [] args) {
+ public static void main(String[] args) {
if (args.length == 0) {
throw new IllegalArgumentException("Missing command. Supported command is remove_corrupt_topologies");
@@ -58,7 +51,7 @@ public class AdminCommands {
removeCorruptTopologies();
break;
default:
- LOG.error("****** "+ command + " is not supported. Supported Admin command - \"remove_corrupt_topologies\" ******");
+ LOG.error("****** " + command + " is not supported. Supported Admin command - \"remove_corrupt_topologies\" ******");
throw new UnsupportedOperationException(command + " is not a supported admin command.");
}
@@ -66,7 +59,7 @@ public class AdminCommands {
private static void initialize() {
conf = Utils.readStormConfig();
- nimbusBlobStore = ServerUtils.getNimbusBlobStore (conf, NimbusInfo.fromConf(conf));
+ nimbusBlobStore = ServerUtils.getNimbusBlobStore(conf, NimbusInfo.fromConf(conf));
try {
stormClusterState = ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf));
} catch (Exception e) {
@@ -75,24 +68,24 @@ public class AdminCommands {
}
}
- private static Set<String> getKeyListFromId( String corruptId) {
+ private static Set<String> getKeyListFromId(String corruptId) {
Set<String> keyLists = new HashSet<>();
keyLists.add(ConfigUtils.masterStormCodeKey(corruptId));
keyLists.add(ConfigUtils.masterStormConfKey(corruptId));
- if(!ConfigUtils.isLocalMode(conf)) {
+ if (!ConfigUtils.isLocalMode(conf)) {
ConfigUtils.masterStormJarKey(corruptId);
}
return keyLists;
}
- private static void removeCorruptTopologies( ) {
+ private static void removeCorruptTopologies() {
Iterator<String> corruptTopologies = listCorruptTopologies();
- while(corruptTopologies.hasNext()) {
+ while (corruptTopologies.hasNext()) {
String corruptId = corruptTopologies.next();
stormClusterState.removeStorm(corruptId);
- if(nimbusBlobStore instanceof LocalFsBlobStore) {
+ if (nimbusBlobStore instanceof LocalFsBlobStore) {
Iterator<String> blobKeys = getKeyListFromId(corruptId).iterator();
- while(blobKeys.hasNext()) {
+ while (blobKeys.hasNext()) {
stormClusterState.removeBlobstoreKey(blobKeys.next());
}
}
@@ -109,7 +102,7 @@ public class AdminCommands {
Set<String> activeTopologyIds = new HashSet<>(stormClusterState.activeStorms());
Sets.SetView<String> diffTopology = Sets.difference(activeTopologyIds, blobStoreTopologyIds);
LOG.info("active-topology-ids [{}] blob-topology-ids [{}] diff-topology [{}]",
- activeTopologyIds.toString(), blobStoreTopologyIds.toString(),diffTopology.toString());
+ activeTopologyIds.toString(), blobStoreTopologyIds.toString(), diffTopology.toString());
return diffTopology.iterator();
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/BasicDrpcClient.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/BasicDrpcClient.java b/storm-core/src/jvm/org/apache/storm/command/BasicDrpcClient.java
index 97fb9d2..eab77c9 100644
--- a/storm-core/src/jvm/org/apache/storm/command/BasicDrpcClient.java
+++ b/storm-core/src/jvm/org/apache/storm/command/BasicDrpcClient.java
@@ -1,29 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.command;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.utils.DRPCClient;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,10 +37,10 @@ public class BasicDrpcClient {
*/
public static void main(String[] args) throws Exception {
Map<String, Object> cl = CLI.opt("f", "function", null, CLI.AS_STRING)
- .arg("FUNC_AND_ARGS", CLI.INTO_LIST)
- .parse(args);
- final List<String> funcAndArgs = (List<String>)cl.get("FUNC_AND_ARGS");
- final String function = (String)cl.get("f");
+ .arg("FUNC_AND_ARGS", CLI.INTO_LIST)
+ .parse(args);
+ final List<String> funcAndArgs = (List<String>) cl.get("FUNC_AND_ARGS");
+ final String function = (String) cl.get("f");
Config conf = new Config();
try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
if (function == null) {
@@ -62,7 +54,7 @@ public class BasicDrpcClient {
runAndPrint(drpc, func, arg);
}
} else {
- for (String arg: funcAndArgs) {
+ for (String arg : funcAndArgs) {
runAndPrint(drpc, function, arg);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/Blobstore.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/Blobstore.java b/storm-core/src/jvm/org/apache/storm/command/Blobstore.java
index c6de51c..d5be3c8 100644
--- a/storm-core/src/jvm/org/apache/storm/command/Blobstore.java
+++ b/storm-core/src/jvm/org/apache/storm/command/Blobstore.java
@@ -1,33 +1,16 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.command;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.storm.blobstore.*;
-import org.apache.storm.generated.AccessControl;
-import org.apache.storm.generated.AuthorizationException;
-import org.apache.storm.generated.KeyNotFoundException;
-import org.apache.storm.generated.ReadableBlobMeta;
-import org.apache.storm.generated.SettableBlobMeta;
-import org.apache.storm.utils.ServerUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+package org.apache.storm.command;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
@@ -42,6 +25,20 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.blobstore.AtomicOutputStream;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.blobstore.BlobStoreAclHandler;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.blobstore.InputStreamWithMeta;
+import org.apache.storm.generated.AccessControl;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.KeyNotFoundException;
+import org.apache.storm.generated.ReadableBlobMeta;
+import org.apache.storm.generated.SettableBlobMeta;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class Blobstore {
private static final Logger LOG = LoggerFactory.getLogger(Blobstore.class);
@@ -90,8 +87,8 @@ public class Blobstore {
private static void readCli(String[] args) throws Exception {
Map<String, Object> cl = CLI.opt("f", "file", null, CLI.AS_STRING)
- .arg("key", CLI.FIRST_WINS)
- .parse(args);
+ .arg("key", CLI.FIRST_WINS)
+ .parse(args);
final String key = (String) cl.get("key");
final String file = (String) cl.get("f");
@@ -106,10 +103,10 @@ public class Blobstore {
private static void createCli(String[] args) throws Exception {
Map<String, Object> cl = CLI.opt("f", "file", null, CLI.AS_STRING)
- .opt("a", "acl", Collections.emptyList(), new AsAclParser())
- .opt("r", "replication-factor", -1, CLI.AS_INT)
- .arg("key", CLI.FIRST_WINS)
- .parse(args);
+ .opt("a", "acl", Collections.emptyList(), new AsAclParser())
+ .opt("r", "replication-factor", -1, CLI.AS_INT)
+ .arg("key", CLI.FIRST_WINS)
+ .parse(args);
final String key = (String) cl.get("key");
final String file = (String) cl.get("f");
@@ -136,8 +133,8 @@ public class Blobstore {
private static void updateCli(String[] args) throws Exception {
Map<String, Object> cl = CLI.opt("f", "file", null, CLI.AS_STRING)
- .arg("key", CLI.FIRST_WINS)
- .parse(args);
+ .arg("key", CLI.FIRST_WINS)
+ .parse(args);
final String key = (String) cl.get("key");
final String file = (String) cl.get("f");
@@ -203,8 +200,8 @@ public class Blobstore {
private static void setAclCli(String[] args) throws Exception {
Map<String, Object> cl = CLI.opt("s", "set", Collections.emptyList(), new AsAclParser())
- .arg("key", CLI.FIRST_WINS)
- .parse(args);
+ .arg("key", CLI.FIRST_WINS)
+ .parse(args);
final String key = (String) cl.get("key");
final List<AccessControl> setAcl = (List<AccessControl>) cl.get("s");
@@ -260,8 +257,8 @@ public class Blobstore {
private void updateReplicationFactor(ClientBlobStore blobStore, String[] args) throws Exception {
Map<String, Object> cl = CLI.opt("r", "replication-factor", null, CLI.AS_INT)
- .arg("key", CLI.FIRST_WINS)
- .parse(args);
+ .arg("key", CLI.FIRST_WINS)
+ .parse(args);
final String key = (String) cl.get("key");
final Integer replicationFactor = (Integer) cl.get("r");
@@ -276,6 +273,14 @@ public class Blobstore {
});
}
+ private static List<String> generateAccessControlsInfo(List<AccessControl> acl) {
+ List<String> accessControlStrings = new ArrayList<>();
+ for (AccessControl ac : acl) {
+ accessControlStrings.add(BlobStoreAclHandler.accessControlToString(ac));
+ }
+ return accessControlStrings;
+ }
+
private static final class BlobStoreSupport {
static void readBlob(final String key, final OutputStream os) throws Exception {
ClientBlobStore.withConfiguredClient(new ClientBlobStore.WithBlobstore() {
@@ -319,14 +324,6 @@ public class Blobstore {
}
}
- private static List<String> generateAccessControlsInfo(List<AccessControl> acl) {
- List<String> accessControlStrings = new ArrayList<>();
- for (AccessControl ac : acl) {
- accessControlStrings.add(BlobStoreAclHandler.accessControlToString(ac));
- }
- return accessControlStrings;
- }
-
private static final class AsAclParser implements CLI.Parse {
@Override
public Object parse(String value) {
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/CLI.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/CLI.java b/storm-core/src/jvm/org/apache/storm/command/CLI.java
index b2afc78..eb62306 100644
--- a/storm-core/src/jvm/org/apache/storm/command/CLI.java
+++ b/storm-core/src/jvm/org/apache/storm/command/CLI.java
@@ -1,22 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.command;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.Option;
@@ -24,56 +23,7 @@ import org.apache.commons.cli.Options;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
public class CLI {
- private static final Logger LOG = LoggerFactory.getLogger(CLI.class);
- private static class Opt {
- final String shortName;
- final String longName;
- final Object defaultValue;
- final Parse parse;
- final Assoc assoc;
- public Opt(String shortName, String longName, Object defaultValue, Parse parse, Assoc assoc) {
- this.shortName = shortName;
- this.longName = longName;
- this.defaultValue = defaultValue;
- this.parse = parse == null ? AS_STRING : parse;
- this.assoc = assoc == null ? LAST_WINS : assoc;
- }
-
- public Object process(Object current, String value) {
- return assoc.assoc(current, parse.parse(value));
- }
- }
-
- private static class Arg {
- final String name;
- final Parse parse;
- final Assoc assoc;
- public Arg(String name, Parse parse, Assoc assoc) {
- this.name = name;
- this.parse = parse == null ? AS_STRING : parse;
- this.assoc = assoc == null ? INTO_LIST : assoc;
- }
-
- public Object process(Object current, String value) {
- return assoc.assoc(current, parse.parse(value));
- }
- }
-
- public interface Parse {
- /**
- * Parse a String to the type you want it to be.
- * @param value the String to parse
- * @return the parsed value
- */
- public Object parse(String value);
- }
-
/**
* Parse function to return an Integer
*/
@@ -83,7 +33,6 @@ public class CLI {
return Integer.valueOf(value);
}
};
-
/**
* Noop parse function, returns the String.
*/
@@ -93,17 +42,6 @@ public class CLI {
return value;
}
};
-
- public interface Assoc {
- /**
- * Associate a value into somthing else
- * @param current what to put value into, will be null if no values have been added yet.
- * @param value what to add
- * @return the result of combining the two
- */
- public Object assoc(Object current, Object value);
- }
-
/**
* Last occurance on the command line is the resulting value.
*/
@@ -113,7 +51,6 @@ public class CLI {
return value;
}
};
-
/**
* First occurance on the command line is the resulting value.
*/
@@ -123,7 +60,6 @@ public class CLI {
return current == null ? value : current;
}
};
-
/**
* All values are returned as a List.
*/
@@ -133,11 +69,10 @@ public class CLI {
if (current == null) {
current = new ArrayList<Object>();
}
- ((List<Object>)current).add(value);
+ ((List<Object>) current).add(value);
return current;
}
};
-
/**
* All values are returned as a map
*/
@@ -151,6 +86,138 @@ public class CLI {
return current;
}
};
+ private static final Logger LOG = LoggerFactory.getLogger(CLI.class);
+
+ /**
+ * Add an option to be parsed
+ * @param shortName the short single character name of the option (no `-` character proceeds it).
+ * @param longName the multi character name of the option (no `--` characters proceed it).
+ * @param defaultValue the value that will be returned of the command if none is given. null if none is given.
+ * @return a builder to be used to continue creating the command line.
+ */
+ public static CLIBuilder opt(String shortName, String longName, Object defaultValue) {
+ return new CLIBuilder().opt(shortName, longName, defaultValue);
+ }
+
+ /**
+ * Add an option to be parsed
+ * @param shortName the short single character name of the option (no `-` character proceeds it).
+ * @param longName the multi character name of the option (no `--` characters proceed it).
+ * @param defaultValue the value that will be returned of the command if none is given. null if none is given.
+ * @param parse an optional function to transform the string to something else. If null a NOOP is used.
+ * @return a builder to be used to continue creating the command line.
+ */
+ public static CLIBuilder opt(String shortName, String longName, Object defaultValue, Parse parse) {
+ return new CLIBuilder().opt(shortName, longName, defaultValue, parse);
+ }
+
+ /**
+ * Add an option to be parsed
+ * @param shortName the short single character name of the option (no `-` character proceeds it).
+ * @param longName the multi character name of the option (no `--` characters proceed it).
+ * @param defaultValue the value that will be returned of the command if none is given. null if none is given.
+ * @param parse an optional function to transform the string to something else. If null a NOOP is used.
+ * @param assoc an association command to decide what to do if the option appears multiple times. If null LAST_WINS is used.
+ * @return a builder to be used to continue creating the command line.
+ */
+ public static CLIBuilder opt(String shortName, String longName, Object defaultValue, Parse parse, Assoc assoc) {
+ return new CLIBuilder().opt(shortName, longName, defaultValue, parse, assoc);
+ }
+
+ /**
+ * Add a named argument.
+ * @param name the name of the argument.
+ * @return a builder to be used to continue creating the command line.
+ */
+ public CLIBuilder arg(String name) {
+ return new CLIBuilder().arg(name);
+ }
+
+ /**
+ * Add a named argument.
+ * @param name the name of the argument.
+ * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used.
+ * @return a builder to be used to continue creating the command line.
+ */
+ public CLIBuilder arg(String name, Assoc assoc) {
+ return new CLIBuilder().arg(name, assoc);
+ }
+
+ /**
+ * Add a named argument.
+ * @param name the name of the argument.
+ * @param parse an optional function to transform the string to something else. If null a NOOP is used.
+ * @return a builder to be used to continue creating the command line.
+ */
+ public CLIBuilder arg(String name, Parse parse) {
+ return new CLIBuilder().arg(name, parse);
+ }
+
+ /**
+ * Add a named argument.
+ * @param name the name of the argument.
+ * @param parse an optional function to transform the string to something else. If null a NOOP is used.
+ * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used.
+ * @return a builder to be used to continue creating the command line.
+ */
+ public CLIBuilder arg(String name, Parse parse, Assoc assoc) {
+ return new CLIBuilder().arg(name, parse, assoc);
+ }
+
+ public interface Parse {
+ /**
+ * Parse a String to the type you want it to be.
+ * @param value the String to parse
+ * @return the parsed value
+ */
+ public Object parse(String value);
+ }
+
+ public interface Assoc {
+ /**
+ * Associate a value into somthing else
+ * @param current what to put value into, will be null if no values have been added yet.
+ * @param value what to add
+ * @return the result of combining the two
+ */
+ public Object assoc(Object current, Object value);
+ }
+
+ private static class Opt {
+ final String shortName;
+ final String longName;
+ final Object defaultValue;
+ final Parse parse;
+ final Assoc assoc;
+
+ public Opt(String shortName, String longName, Object defaultValue, Parse parse, Assoc assoc) {
+ this.shortName = shortName;
+ this.longName = longName;
+ this.defaultValue = defaultValue;
+ this.parse = parse == null ? AS_STRING : parse;
+ this.assoc = assoc == null ? LAST_WINS : assoc;
+ }
+
+ public Object process(Object current, String value) {
+ return assoc.assoc(current, parse.parse(value));
+ }
+ }
+
+ private static class Arg {
+ final String name;
+ final Parse parse;
+ final Assoc assoc;
+
+ public Arg(String name, Parse parse, Assoc assoc) {
+ this.name = name;
+ this.parse = parse == null ? AS_STRING : parse;
+ this.assoc = assoc == null ? INTO_LIST : assoc;
+ }
+
+ public Object process(Object current, String value) {
+ return assoc.assoc(current, parse.parse(value));
+ }
+ }
public static class CLIBuilder {
private final ArrayList<Opt> opts = new ArrayList<>();
@@ -243,9 +310,9 @@ public class CLI {
* args will be stored under the argument name, unless no arguments are configured, and then they will be stored under "ARGS".
* The last argument comnfigured is greedy and is used to process all remaining command line arguments.
*/
- public Map<String, Object> parse(String ... rawArgs) throws Exception {
+ public Map<String, Object> parse(String... rawArgs) throws Exception {
Options options = new Options();
- for (Opt opt: opts) {
+ for (Opt opt : opts) {
options.addOption(Option.builder(opt.shortName).longOpt(opt.longName).hasArg().build());
}
DefaultParser parser = new DefaultParser();
@@ -278,7 +345,7 @@ public class CLI {
boolean isLastArg = (argIndex == (args.size() - 1));
Object current = null;
int maxStringIndex = isLastArg ? stringArgs.size() : (stringArgIndex + 1);
- for (;stringArgIndex < maxStringIndex; stringArgIndex++) {
+ for (; stringArgIndex < maxStringIndex; stringArgIndex++) {
current = arg.process(current, stringArgs.get(stringArgIndex));
}
ret.put(arg.name, current);
@@ -290,80 +357,4 @@ public class CLI {
return ret;
}
}
-
- /**
- * Add an option to be parsed
- * @param shortName the short single character name of the option (no `-` character proceeds it).
- * @param longName the multi character name of the option (no `--` characters proceed it).
- * @param defaultValue the value that will be returned of the command if none is given. null if none is given.
- * @return a builder to be used to continue creating the command line.
- */
- public static CLIBuilder opt(String shortName, String longName, Object defaultValue) {
- return new CLIBuilder().opt(shortName, longName, defaultValue);
- }
-
- /**
- * Add an option to be parsed
- * @param shortName the short single character name of the option (no `-` character proceeds it).
- * @param longName the multi character name of the option (no `--` characters proceed it).
- * @param defaultValue the value that will be returned of the command if none is given. null if none is given.
- * @param parse an optional function to transform the string to something else. If null a NOOP is used.
- * @return a builder to be used to continue creating the command line.
- */
- public static CLIBuilder opt(String shortName, String longName, Object defaultValue, Parse parse) {
- return new CLIBuilder().opt(shortName, longName, defaultValue, parse);
- }
-
- /**
- * Add an option to be parsed
- * @param shortName the short single character name of the option (no `-` character proceeds it).
- * @param longName the multi character name of the option (no `--` characters proceed it).
- * @param defaultValue the value that will be returned of the command if none is given. null if none is given.
- * @param parse an optional function to transform the string to something else. If null a NOOP is used.
- * @param assoc an association command to decide what to do if the option appears multiple times. If null LAST_WINS is used.
- * @return a builder to be used to continue creating the command line.
- */
- public static CLIBuilder opt(String shortName, String longName, Object defaultValue, Parse parse, Assoc assoc) {
- return new CLIBuilder().opt(shortName, longName, defaultValue, parse, assoc);
- }
-
- /**
- * Add a named argument.
- * @param name the name of the argument.
- * @return a builder to be used to continue creating the command line.
- */
- public CLIBuilder arg(String name) {
- return new CLIBuilder().arg(name);
- }
-
- /**
- * Add a named argument.
- * @param name the name of the argument.
- * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used.
- * @return a builder to be used to continue creating the command line.
- */
- public CLIBuilder arg(String name, Assoc assoc) {
- return new CLIBuilder().arg(name, assoc);
- }
-
- /**
- * Add a named argument.
- * @param name the name of the argument.
- * @param parse an optional function to transform the string to something else. If null a NOOP is used.
- * @return a builder to be used to continue creating the command line.
- */
- public CLIBuilder arg(String name, Parse parse) {
- return new CLIBuilder().arg(name, parse);
- }
-
- /**
- * Add a named argument.
- * @param name the name of the argument.
- * @param parse an optional function to transform the string to something else. If null a NOOP is used.
- * @param assoc an association command to decide what to do if the argument appears multiple times. If null INTO_LIST is used.
- * @return a builder to be used to continue creating the command line.
- */
- public CLIBuilder arg(String name, Parse parse, Assoc assoc) {
- return new CLIBuilder().arg(name, parse, assoc);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/ConfigValue.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/ConfigValue.java b/storm-core/src/jvm/org/apache/storm/command/ConfigValue.java
index 6b0ac46..eb6e451 100644
--- a/storm-core/src/jvm/org/apache/storm/command/ConfigValue.java
+++ b/storm-core/src/jvm/org/apache/storm/command/ConfigValue.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.command;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-
import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.Utils;
@@ -45,9 +39,9 @@ public final class ConfigValue {
Object value = conf.get(name);
if (value instanceof List) {
List<String> stringValues = ((List<?>) value)
- .stream()
- .map(ObjectReader::getString)
- .collect(Collectors.toList());
+ .stream()
+ .map(ObjectReader::getString)
+ .collect(Collectors.toList());
value = String.join(" ", stringValues);
}
System.out.println("VALUE: " + value);
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/Deactivate.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/Deactivate.java b/storm-core/src/jvm/org/apache/storm/command/Deactivate.java
index 4eafb83..d56f6ff 100644
--- a/storm-core/src/jvm/org/apache/storm/command/Deactivate.java
+++ b/storm-core/src/jvm/org/apache/storm/command/Deactivate.java
@@ -1,40 +1,34 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.command;
import org.apache.storm.generated.Nimbus;
import org.apache.storm.utils.NimbusClient;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Deactivate {
private static final Logger LOG = LoggerFactory.getLogger(Deactivate.class);
- public static void main(String [] args) throws Exception {
+ public static void main(String[] args) throws Exception {
final String name = args[0];
NimbusClient.withConfiguredClient(new NimbusClient.WithNimbus() {
- @Override
- public void run(Nimbus.Iface nimbus) throws Exception {
- nimbus.deactivate(name);
- LOG.info("Deactivated topology: {}", name);
- }
+ @Override
+ public void run(Nimbus.Iface nimbus) throws Exception {
+ nimbus.deactivate(name);
+ LOG.info("Deactivated topology: {}", name);
+ }
});
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/DevZookeeper.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/DevZookeeper.java b/storm-core/src/jvm/org/apache/storm/command/DevZookeeper.java
index ef53170..7e98e0d 100644
--- a/storm-core/src/jvm/org/apache/storm/command/DevZookeeper.java
+++ b/storm-core/src/jvm/org/apache/storm/command/DevZookeeper.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.command;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
import org.apache.storm.utils.ObjectReader;
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/GetErrors.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/GetErrors.java b/storm-core/src/jvm/org/apache/storm/command/GetErrors.java
index 05c0ee4..dafd2cb 100644
--- a/storm-core/src/jvm/org/apache/storm/command/GetErrors.java
+++ b/storm-core/src/jvm/org/apache/storm/command/GetErrors.java
@@ -1,35 +1,29 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.command;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.generated.ErrorInfo;
import org.apache.storm.generated.GetInfoOptions;
import org.apache.storm.generated.Nimbus;
import org.apache.storm.generated.NumErrorsChoice;
import org.apache.storm.generated.TopologyInfo;
-import org.apache.storm.utils.Utils;
import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
import org.json.simple.JSONValue;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
public class GetErrors {
public static void main(String[] args) throws Exception {
if (args.length == 0) {
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java b/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java
index 6cfde2c..ce7f53e 100644
--- a/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java
+++ b/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.command;
import java.util.Map;
-
import org.apache.storm.healthcheck.HealthChecker;
import org.apache.storm.utils.Utils;
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/Heartbeats.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/Heartbeats.java b/storm-core/src/jvm/org/apache/storm/command/Heartbeats.java
index 04dc954..4236d4d 100644
--- a/storm-core/src/jvm/org/apache/storm/command/Heartbeats.java
+++ b/storm-core/src/jvm/org/apache/storm/command/Heartbeats.java
@@ -1,24 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.command;
+import com.google.common.base.Joiner;
import java.util.Map;
-
import org.apache.storm.cluster.ClusterStateContext;
import org.apache.storm.cluster.ClusterUtils;
import org.apache.storm.cluster.IStateStorage;
@@ -29,8 +24,6 @@ import org.json.simple.JSONValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Joiner;
-
public class Heartbeats {
private static final Logger LOG = LoggerFactory.getLogger(Heartbeats.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/KillTopology.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/KillTopology.java b/storm-core/src/jvm/org/apache/storm/command/KillTopology.java
index b8f9a91..696c199 100644
--- a/storm-core/src/jvm/org/apache/storm/command/KillTopology.java
+++ b/storm-core/src/jvm/org/apache/storm/command/KillTopology.java
@@ -1,54 +1,47 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.command;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.generated.KillOptions;
import org.apache.storm.generated.Nimbus;
import org.apache.storm.utils.NimbusClient;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class KillTopology {
private static final Logger LOG = LoggerFactory.getLogger(KillTopology.class);
- public static void main(String [] args) throws Exception {
+ public static void main(String[] args) throws Exception {
Map<String, Object> cl = CLI.opt("w", "wait", null, CLI.AS_INT)
.arg("TOPO", CLI.INTO_LIST)
.parse(args);
- final List<String> names = (List<String>)cl.get("TOPO");
- Integer wait = (Integer)cl.get("w");
+ final List<String> names = (List<String>) cl.get("TOPO");
+ Integer wait = (Integer) cl.get("w");
final KillOptions opts = new KillOptions();
if (wait != null) {
opts.set_wait_secs(wait);
}
NimbusClient.withConfiguredClient(new NimbusClient.WithNimbus() {
- @Override
- public void run(Nimbus.Iface nimbus) throws Exception {
- for (String name: names) {
- nimbus.killTopologyWithOpts(name, opts);
- LOG.info("Killed topology: {}", name);
+ @Override
+ public void run(Nimbus.Iface nimbus) throws Exception {
+ for (String name : names) {
+ nimbus.killTopologyWithOpts(name, opts);
+ LOG.info("Killed topology: {}", name);
+ }
}
- }
});
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java b/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java
index 5dc8402..c417ace 100644
--- a/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java
+++ b/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java
@@ -1,34 +1,28 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.command;
import java.io.File;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.daemon.supervisor.StandaloneSupervisor;
import org.apache.storm.daemon.supervisor.Supervisor;
import org.apache.storm.utils.Utils;
public class KillWorkers {
- public static void main(String [] args) throws Exception {
+ public static void main(String[] args) throws Exception {
Map<String, Object> conf = Utils.readStormConfig();
- conf.put(Config.STORM_LOCAL_DIR, new File((String)conf.get(Config.STORM_LOCAL_DIR)).getCanonicalPath());
+ conf.put(Config.STORM_LOCAL_DIR, new File((String) conf.get(Config.STORM_LOCAL_DIR)).getCanonicalPath());
try (Supervisor supervisor = new Supervisor(conf, null, new StandaloneSupervisor())) {
supervisor.shutdownAllWorkers(null, null);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/ListTopologies.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/ListTopologies.java b/storm-core/src/jvm/org/apache/storm/command/ListTopologies.java
index 20d5b6f..4fe2982 100644
--- a/storm-core/src/jvm/org/apache/storm/command/ListTopologies.java
+++ b/storm-core/src/jvm/org/apache/storm/command/ListTopologies.java
@@ -1,52 +1,45 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.command;
+import java.util.List;
import org.apache.storm.generated.Nimbus;
import org.apache.storm.generated.TopologySummary;
import org.apache.storm.utils.NimbusClient;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.List;
-
public class ListTopologies {
private static final Logger LOG = LoggerFactory.getLogger(ListTopologies.class);
private static final String MSG_FORMAT = "%-20s %-10s %-10s %-12s %-12s %-20s\n";
- public static void main(String [] args) throws Exception {
+ public static void main(String[] args) throws Exception {
NimbusClient.withConfiguredClient(new NimbusClient.WithNimbus() {
- @Override
- public void run(Nimbus.Iface nimbus) throws Exception {
- List<TopologySummary> topologies = nimbus.getClusterInfo().get_topologies();
- if (topologies == null || topologies.isEmpty()) {
- System.out.println("No topologies running.");
- } else {
- System.out.printf(MSG_FORMAT, "Topology_name", "Status", "Num_tasks", "Num_workers", "Uptime_secs", "Topology_Id");
- System.out.println("----------------------------------------------------------------------------------------");
- for (TopologySummary topology: topologies) {
- System.out.printf(MSG_FORMAT, topology.get_name(), topology.get_status(),
- topology.get_num_tasks(), topology.get_num_workers(),
- topology.get_uptime_secs(), topology.get_id());
- }
- }
- }
+ @Override
+ public void run(Nimbus.Iface nimbus) throws Exception {
+ List<TopologySummary> topologies = nimbus.getClusterInfo().get_topologies();
+ if (topologies == null || topologies.isEmpty()) {
+ System.out.println("No topologies running.");
+ } else {
+ System.out.printf(MSG_FORMAT, "Topology_name", "Status", "Num_tasks", "Num_workers", "Uptime_secs", "Topology_Id");
+ System.out.println("----------------------------------------------------------------------------------------");
+ for (TopologySummary topology : topologies) {
+ System.out.printf(MSG_FORMAT, topology.get_name(), topology.get_status(),
+ topology.get_num_tasks(), topology.get_num_workers(),
+ topology.get_uptime_secs(), topology.get_id());
+ }
+ }
+ }
});
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/Monitor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/Monitor.java b/storm-core/src/jvm/org/apache/storm/command/Monitor.java
index c4da6c8..7ade55e 100644
--- a/storm-core/src/jvm/org/apache/storm/command/Monitor.java
+++ b/storm-core/src/jvm/org/apache/storm/command/Monitor.java
@@ -1,37 +1,30 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.command;
+import java.util.Map;
import org.apache.storm.generated.Nimbus;
import org.apache.storm.utils.NimbusClient;
-import java.util.Map;
-
public class Monitor {
public static void main(String[] args) throws Exception {
Map<String, Object> cl = CLI.opt("i", "interval", 4, CLI.AS_INT)
- .opt("m", "component", null)
- .opt("s", "stream", "default")
- .opt("w", "watch", "emitted")
- .arg("topologyName", CLI.FIRST_WINS)
- .parse(args);
+ .opt("m", "component", null)
+ .opt("s", "stream", "default")
+ .opt("w", "watch", "emitted")
+ .arg("topologyName", CLI.FIRST_WINS)
+ .parse(args);
final org.apache.storm.utils.Monitor monitor = new org.apache.storm.utils.Monitor();
Integer interval = (Integer) cl.get("i");
String component = (String) cl.get("m");
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/Rebalance.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/Rebalance.java b/storm-core/src/jvm/org/apache/storm/command/Rebalance.java
index ec34798..4ecf4c9 100644
--- a/storm-core/src/jvm/org/apache/storm/command/Rebalance.java
+++ b/storm-core/src/jvm/org/apache/storm/command/Rebalance.java
@@ -1,23 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.command;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.storm.generated.Nimbus;
import org.apache.storm.generated.RebalanceOptions;
import org.apache.storm.utils.NimbusClient;
@@ -26,9 +22,6 @@ import org.json.simple.JSONValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.Map;
-
import static java.lang.String.format;
public class Rebalance {
@@ -37,12 +30,12 @@ public class Rebalance {
public static void main(String[] args) throws Exception {
Map<String, Object> cl = CLI.opt("w", "wait", null, CLI.AS_INT)
- .opt("n", "num-workers", null, CLI.AS_INT)
- .opt("e", "executor", null, new ExecutorParser(), CLI.INTO_MAP)
- .opt("r", "resources", null, new ResourcesParser(), CLI.INTO_MAP)
- .opt("t", "topology-conf", null, new ConfParser(), CLI.INTO_MAP)
- .arg("topologyName", CLI.FIRST_WINS)
- .parse(args);
+ .opt("n", "num-workers", null, CLI.AS_INT)
+ .opt("e", "executor", null, new ExecutorParser(), CLI.INTO_MAP)
+ .opt("r", "resources", null, new ResourcesParser(), CLI.INTO_MAP)
+ .opt("t", "topology-conf", null, new ConfParser(), CLI.INTO_MAP)
+ .arg("topologyName", CLI.FIRST_WINS)
+ .parse(args);
final String name = (String) cl.get("topologyName");
final RebalanceOptions rebalanceOptions = new RebalanceOptions();
Integer wait = (Integer) cl.get("w");
@@ -104,11 +97,11 @@ public class Rebalance {
// But because value is coming from JSON it is going to be a Number, and we want it to be a Double.
// So the goal is to go through each entry and update it accordingly
Map<String, Map<String, Double>> ret = new HashMap<>();
- for (Map.Entry<String, Object> compEntry: Utils.parseJson(value).entrySet()) {
+ for (Map.Entry<String, Object> compEntry : Utils.parseJson(value).entrySet()) {
String comp = compEntry.getKey();
Map<String, Number> numResources = (Map<String, Number>) compEntry.getValue();
Map<String, Double> doubleResource = new HashMap<>();
- for (Map.Entry<String, Number> entry: numResources.entrySet()) {
+ for (Map.Entry<String, Number> entry : numResources.entrySet()) {
doubleResource.put(entry.getKey(), entry.getValue().doubleValue());
}
ret.put(comp, doubleResource);
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/SetLogLevel.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/SetLogLevel.java b/storm-core/src/jvm/org/apache/storm/command/SetLogLevel.java
index 75708a8..24f04e1 100644
--- a/storm-core/src/jvm/org/apache/storm/command/SetLogLevel.java
+++ b/storm-core/src/jvm/org/apache/storm/command/SetLogLevel.java
@@ -1,47 +1,39 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.command;
import com.google.common.base.Preconditions;
-
+import java.util.HashMap;
+import java.util.Map;
import org.apache.logging.log4j.Level;
import org.apache.storm.generated.LogConfig;
import org.apache.storm.generated.LogLevel;
import org.apache.storm.generated.LogLevelAction;
import org.apache.storm.generated.Nimbus;
-import org.apache.storm.utils.Utils;
import org.apache.storm.utils.NimbusClient;
+import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.Map;
-
public class SetLogLevel {
private static final Logger LOG = LoggerFactory.getLogger(SetLogLevel.class);
public static void main(String[] args) throws Exception {
Map<String, Object> cl = CLI.opt("l", "log-setting", null, new LogLevelsParser(LogLevelAction.UPDATE), CLI.INTO_MAP)
- .opt("r", "remove-log-setting", null, new LogLevelsParser(LogLevelAction.REMOVE), CLI.INTO_MAP)
- .arg("topologyName", CLI.FIRST_WINS)
- .parse(args);
+ .opt("r", "remove-log-setting", null, new LogLevelsParser(LogLevelAction.REMOVE), CLI.INTO_MAP)
+ .arg("topologyName", CLI.FIRST_WINS)
+ .parse(args);
final String topologyName = (String) cl.get("topologyName");
final LogConfig logConfig = new LogConfig();
Map<String, LogLevel> logLevelMap = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/command/UploadCredentials.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/UploadCredentials.java b/storm-core/src/jvm/org/apache/storm/command/UploadCredentials.java
index a72039b..9a71fac 100644
--- a/storm-core/src/jvm/org/apache/storm/command/UploadCredentials.java
+++ b/storm-core/src/jvm/org/apache/storm/command/UploadCredentials.java
@@ -1,31 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.command;
-import org.apache.storm.StormSubmitter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+package org.apache.storm.command;
import java.io.FileReader;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import org.apache.storm.StormSubmitter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class UploadCredentials {
@@ -33,9 +27,9 @@ public class UploadCredentials {
public static void main(String[] args) throws Exception {
Map<String, Object> cl = CLI.opt("f", "file", null)
- .arg("topologyName", CLI.FIRST_WINS)
- .arg("rawCredentials", CLI.INTO_LIST)
- .parse(args);
+ .arg("topologyName", CLI.FIRST_WINS)
+ .arg("rawCredentials", CLI.INTO_LIST)
+ .parse(args);
String credentialFile = (String) cl.get("f");
List<String> rawCredentials = (List<String>) cl.get("rawCredentials");
@@ -44,13 +38,13 @@ public class UploadCredentials {
if (null != rawCredentials && ((rawCredentials.size() % 2) != 0)) {
throw new RuntimeException("Need an even number of arguments to make a map");
}
- Map<String,String> credentialsMap = new HashMap<>();
+ Map<String, String> credentialsMap = new HashMap<>();
if (null != credentialFile) {
Properties credentialProps = new Properties();
credentialProps.load(new FileReader(credentialFile));
- for(Map.Entry<Object, Object> credentialProp : credentialProps.entrySet()) {
- credentialsMap.put((String)credentialProp.getKey(),
- (String)credentialProp.getValue());
+ for (Map.Entry<Object, Object> credentialProp : credentialProps.entrySet()) {
+ credentialsMap.put((String) credentialProp.getKey(),
+ (String) credentialProp.getValue());
}
}
if (null != rawCredentials) {
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/planner/CompoundSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/planner/CompoundSpout.java b/storm-core/src/jvm/org/apache/storm/planner/CompoundSpout.java
index 65d546e..2a46f27 100644
--- a/storm-core/src/jvm/org/apache/storm/planner/CompoundSpout.java
+++ b/storm-core/src/jvm/org/apache/storm/planner/CompoundSpout.java
@@ -1,25 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.planner;
public class CompoundSpout
- //implements ISpout
+ //implements ISpout
{
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/planner/CompoundTask.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/planner/CompoundTask.java b/storm-core/src/jvm/org/apache/storm/planner/CompoundTask.java
index 13bcfb5..c9fe535 100644
--- a/storm-core/src/jvm/org/apache/storm/planner/CompoundTask.java
+++ b/storm-core/src/jvm/org/apache/storm/planner/CompoundTask.java
@@ -1,25 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.planner;
public class CompoundTask
-// implements IBolt
+ // implements IBolt
{
}
[51/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-hbase-examples
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-hbase-examples
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f4ba7c95
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f4ba7c95
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f4ba7c95
Branch: refs/heads/master
Commit: f4ba7c952825ef7b0ee10bd1861dd0f288fe7761
Parents: 880d14f
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 23:11:24 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 02:32:25 2018 -0400
----------------------------------------------------------------------
examples/storm-hbase-examples/pom.xml | 5 +-
.../storm/hbase/topology/LookupWordCount.java | 28 ++++------
.../hbase/topology/PersistentWordCount.java | 36 ++++++-------
.../storm/hbase/topology/TotalWordCounter.java | 31 +++++------
.../storm/hbase/topology/WordCountClient.java | 21 +++-----
.../hbase/topology/WordCountValueMapper.java | 43 +++++++--------
.../storm/hbase/topology/WordCounter.java | 22 +++-----
.../apache/storm/hbase/topology/WordSpout.java | 28 ++++------
.../storm/hbase/trident/PrintFunction.java | 28 ++++------
.../storm/hbase/trident/WordCountTrident.java | 57 +++++++++-----------
10 files changed, 124 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f4ba7c95/examples/storm-hbase-examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/pom.xml b/examples/storm-hbase-examples/pom.xml
index 458d71d..16953e2 100644
--- a/examples/storm-hbase-examples/pom.xml
+++ b/examples/storm-hbase-examples/pom.xml
@@ -15,7 +15,8 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -89,7 +90,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>55</maxAllowedViolations>
+ <maxAllowedViolations>16</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/f4ba7c95/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/LookupWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/LookupWordCount.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/LookupWordCount.java
index a10f34f..1c365c4 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/LookupWordCount.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/LookupWordCount.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.topology;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.hbase.bolt.HBaseLookupBolt;
@@ -38,7 +32,7 @@ public class LookupWordCount {
Config config = new Config();
Map<String, Object> hbConf = new HashMap<String, Object>();
- if(args.length > 0){
+ if (args.length > 0) {
hbConf.put("hbase.rootdir", args[0]);
}
config.put("hbase.conf", hbConf);
@@ -53,8 +47,8 @@ public class LookupWordCount {
WordCountValueMapper rowToTupleMapper = new WordCountValueMapper();
HBaseLookupBolt hBaseLookupBolt = new HBaseLookupBolt("WordCount", mapper, rowToTupleMapper)
- .withConfigKey("hbase.conf")
- .withProjectionCriteria(projectionCriteria);
+ .withConfigKey("hbase.conf")
+ .withProjectionCriteria(projectionCriteria);
//wordspout -> lookupbolt -> totalCountBolt
TopologyBuilder builder = new TopologyBuilder();
@@ -68,7 +62,7 @@ public class LookupWordCount {
System.out.println("Usage: LookupWordCount <hbase.rootdir>");
return;
}
-
+
StormSubmitter.submitTopology(topoName, config, builder.createTopology());
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f4ba7c95/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/PersistentWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/PersistentWordCount.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/PersistentWordCount.java
index 700a7cc..65de922 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/PersistentWordCount.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/PersistentWordCount.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.topology;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.hbase.bolt.HBaseBolt;
@@ -40,7 +34,7 @@ public class PersistentWordCount {
Config config = new Config();
Map<String, Object> hbConf = new HashMap<String, Object>();
- if(args.length > 0){
+ if (args.length > 0) {
hbConf.put("hbase.rootdir", args[0]);
}
config.put("hbase.conf", hbConf);
@@ -49,13 +43,13 @@ public class PersistentWordCount {
WordCounter bolt = new WordCounter();
SimpleHBaseMapper mapper = new SimpleHBaseMapper()
- .withRowKeyField("word")
- .withColumnFields(new Fields("word"))
- .withCounterFields(new Fields("count"))
- .withColumnFamily("cf");
+ .withRowKeyField("word")
+ .withColumnFields(new Fields("word"))
+ .withCounterFields(new Fields("count"))
+ .withColumnFamily("cf");
HBaseBolt hbase = new HBaseBolt("WordCount", mapper)
- .withConfigKey("hbase.conf");
+ .withConfigKey("hbase.conf");
// wordSpout ==> countBolt ==> HBaseBolt
@@ -69,8 +63,8 @@ public class PersistentWordCount {
if (args.length == 2) {
topoName = args[0];
} else if (args.length == 4) {
- System.out.println("hdfs url: " + args[0] + ", keytab file: " + args[2] +
- ", principal name: " + args[3] + ", toplogy name: " + args[1]);
+ System.out.println("hdfs url: " + args[0] + ", keytab file: " + args[2] +
+ ", principal name: " + args[3] + ", toplogy name: " + args[1]);
hbConf.put(HBaseSecurityUtil.STORM_KEYTAB_FILE_KEY, args[2]);
hbConf.put(HBaseSecurityUtil.STORM_USER_NAME_KEY, args[3]);
config.setNumWorkers(3);
http://git-wip-us.apache.org/repos/asf/storm/blob/f4ba7c95/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/TotalWordCounter.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/TotalWordCounter.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/TotalWordCounter.java
index c107282..313340a 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/TotalWordCounter.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/TotalWordCounter.java
@@ -1,22 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.topology;
+import java.math.BigInteger;
+import java.util.Map;
+import java.util.Random;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.IBasicBolt;
@@ -26,17 +24,14 @@ import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.math.BigInteger;
-import java.util.Map;
-import java.util.Random;
-
import static org.apache.storm.utils.Utils.tuple;
public class TotalWordCounter implements IBasicBolt {
- private BigInteger total = BigInteger.ZERO;
private static final Logger LOG = LoggerFactory.getLogger(TotalWordCounter.class);
private static final Random RANDOM = new Random();
+ private BigInteger total = BigInteger.ZERO;
+
public void prepare(Map<String, Object> topoConf, TopologyContext context) {
}
@@ -48,7 +43,7 @@ public class TotalWordCounter implements IBasicBolt {
total = total.add(new BigInteger(input.getValues().get(1).toString()));
collector.emit(tuple(total.toString()));
//prints the total with low probability.
- if(RANDOM.nextInt(1000) > 995) {
+ if (RANDOM.nextInt(1000) > 995) {
LOG.info("Running total = " + total);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f4ba7c95/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCountClient.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCountClient.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCountClient.java
index 33ce450..b7874bd 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCountClient.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCountClient.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.topology;
import org.apache.hadoop.conf.Configuration;
@@ -33,7 +28,7 @@ public class WordCountClient {
public static void main(String[] args) throws Exception {
Configuration config = HBaseConfiguration.create();
- if(args.length > 0){
+ if (args.length > 0) {
config.set("hbase.rootdir", args[0]);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f4ba7c95/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCountValueMapper.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCountValueMapper.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCountValueMapper.java
index 6c3301b..15ece78 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCountValueMapper.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCountValueMapper.java
@@ -1,39 +1,33 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.topology;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.ITuple;
-import org.apache.storm.tuple.Values;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.storm.hbase.bolt.mapper.HBaseValueMapper;
-
-import java.util.ArrayList;
-import java.util.List;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.ITuple;
+import org.apache.storm.tuple.Values;
/**
- * Takes a Hbase result and returns a value list that has a value instance for each column and corresponding value.
- * So if the result from Hbase was
+ * Takes a Hbase result and returns a value list that has a value instance for each column and corresponding value. So if the result from
+ * Hbase was
* <pre>
* WORD, COUNT
* apple, 10
@@ -47,7 +41,6 @@ import java.util.List;
* [WORD, banana]
* [COUNT, 20]
* </pre>
- *
*/
public class WordCountValueMapper implements HBaseValueMapper {
@@ -55,8 +48,8 @@ public class WordCountValueMapper implements HBaseValueMapper {
public List<Values> toValues(ITuple tuple, Result result) throws Exception {
List<Values> values = new ArrayList<Values>();
Cell[] cells = result.rawCells();
- for(Cell cell : cells) {
- Values value = new Values (Bytes.toString(CellUtil.cloneQualifier(cell)), Bytes.toLong(CellUtil.cloneValue(cell)));
+ for (Cell cell : cells) {
+ Values value = new Values(Bytes.toString(CellUtil.cloneQualifier(cell)), Bytes.toLong(CellUtil.cloneValue(cell)));
values.add(value);
}
return values;
@@ -64,7 +57,7 @@ public class WordCountValueMapper implements HBaseValueMapper {
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("columnName","columnValue"));
+ declarer.declare(new Fields("columnName", "columnValue"));
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f4ba7c95/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCounter.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCounter.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCounter.java
index 5c71ade..933cc49 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCounter.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordCounter.java
@@ -1,22 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.topology;
+import java.util.Map;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.IBasicBolt;
@@ -24,8 +20,6 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
-import java.util.Map;
-
import static org.apache.storm.utils.Utils.tuple;
public class WordCounter implements IBasicBolt {
http://git-wip-us.apache.org/repos/asf/storm/blob/f4ba7c95/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordSpout.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordSpout.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordSpout.java
index bcc9642..3e407d6 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordSpout.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/topology/WordSpout.java
@@ -1,22 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.topology;
+import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.IRichSpout;
@@ -24,14 +22,10 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-
public class WordSpout implements IRichSpout {
+ public static final String[] words = new String[]{ "apple", "orange", "pineapple", "banana", "watermelon" };
boolean isDistributed;
SpoutOutputCollector collector;
- public static final String[] words = new String[] { "apple", "orange", "pineapple", "banana", "watermelon" };
public WordSpout() {
this(true);
http://git-wip-us.apache.org/repos/asf/storm/blob/f4ba7c95/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/PrintFunction.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/PrintFunction.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/PrintFunction.java
index cdc7690..7285e54 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/PrintFunction.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/PrintFunction.java
@@ -1,29 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.trident;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.Random;
import org.apache.storm.trident.operation.BaseFunction;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.tuple.TridentTuple;
-
-import java.util.Random;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class PrintFunction extends BaseFunction {
@@ -33,7 +27,7 @@ public class PrintFunction extends BaseFunction {
@Override
public void execute(TridentTuple tuple, TridentCollector tridentCollector) {
- if(RANDOM.nextInt(1000) > 995) {
+ if (RANDOM.nextInt(1000) > 995) {
LOG.info(tuple.toString());
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f4ba7c95/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/WordCountTrident.java
----------------------------------------------------------------------
diff --git a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/WordCountTrident.java b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/WordCountTrident.java
index c81512e..4861765 100644
--- a/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/WordCountTrident.java
+++ b/examples/storm-hbase-examples/src/main/java/org/apache/storm/hbase/trident/WordCountTrident.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.trident;
import org.apache.hadoop.hbase.client.Durability;
@@ -39,21 +34,21 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
public class WordCountTrident {
- public static StormTopology buildTopology(String hbaseRoot){
+ public static StormTopology buildTopology(String hbaseRoot) {
Fields fields = new Fields("word", "count");
FixedBatchSpout spout = new FixedBatchSpout(fields, 4,
- new Values("storm", 1),
- new Values("trident", 1),
- new Values("needs", 1),
- new Values("javadoc", 1)
+ new Values("storm", 1),
+ new Values("trident", 1),
+ new Values("needs", 1),
+ new Values("javadoc", 1)
);
spout.setCycle(true);
TridentHBaseMapper tridentHBaseMapper = new SimpleTridentHBaseMapper()
- .withColumnFamily("cf")
- .withColumnFields(new Fields("word"))
- .withCounterFields(new Fields("count"))
- .withRowKeyField("word");
+ .withColumnFamily("cf")
+ .withColumnFields(new Fields("word"))
+ .withCounterFields(new Fields("count"))
+ .withRowKeyField("word");
HBaseValueMapper rowToStormValueMapper = new WordCountValueMapper();
@@ -61,23 +56,23 @@ public class WordCountTrident {
projectionCriteria.addColumn(new HBaseProjectionCriteria.ColumnMetaData("cf", "count"));
HBaseState.Options options = new HBaseState.Options()
- .withConfigKey(hbaseRoot)
- .withDurability(Durability.SYNC_WAL)
- .withMapper(tridentHBaseMapper)
- .withProjectionCriteria(projectionCriteria)
- .withRowToStormValueMapper(rowToStormValueMapper)
- .withTableName("WordCount");
+ .withConfigKey(hbaseRoot)
+ .withDurability(Durability.SYNC_WAL)
+ .withMapper(tridentHBaseMapper)
+ .withProjectionCriteria(projectionCriteria)
+ .withRowToStormValueMapper(rowToStormValueMapper)
+ .withTableName("WordCount");
StateFactory factory = new HBaseStateFactory(options);
TridentTopology topology = new TridentTopology();
Stream stream = topology.newStream("spout1", spout);
- stream.partitionPersist(factory, fields, new HBaseUpdater(), new Fields());
+ stream.partitionPersist(factory, fields, new HBaseUpdater(), new Fields());
TridentState state = topology.newStaticState(factory);
- stream = stream.stateQuery(state, new Fields("word"), new HBaseQuery(), new Fields("columnName","columnValue"));
- stream.each(new Fields("word","columnValue"), new PrintFunction(), new Fields());
+ stream = stream.stateQuery(state, new Fields("word"), new HBaseQuery(), new Fields("columnName", "columnValue"));
+ stream.each(new Fields("word", "columnValue"), new PrintFunction(), new Fields());
return topology.build();
}
@@ -85,7 +80,7 @@ public class WordCountTrident {
Config conf = new Config();
conf.setMaxSpoutPending(5);
String topoName = "wordCounter";
-
+
if (args.length == 2) {
topoName = args[1];
} else if (args.length > 2) {
[42/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-starter
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-starter
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/81ec15d1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/81ec15d1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/81ec15d1
Branch: refs/heads/master
Commit: 81ec15d1096cd526b94313661e7b5de7ed1791d0
Parents: 7da98cf
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 22:36:19 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 00:22:36 2018 -0400
----------------------------------------------------------------------
examples/storm-starter/pom.xml | 2 +-
.../apache/storm/starter/AnchoredWordCount.java | 72 +-
.../apache/storm/starter/BasicDRPCTopology.java | 49 +-
.../starter/BlobStoreAPIWordCountTopology.java | 276 ++++----
.../storm/starter/ExclamationTopology.java | 84 ++-
.../storm/starter/FastWordCountTopology.java | 300 ++++-----
.../storm/starter/InOrderDeliveryTest.java | 261 ++++----
.../apache/storm/starter/JoinBoltExample.java | 39 +-
.../apache/storm/starter/LambdaTopology.java | 24 +-
.../org/apache/storm/starter/ManualDRPC.java | 51 +-
.../storm/starter/MultipleLoggerTopology.java | 112 ++--
.../starter/PersistentWindowingTopology.java | 99 ++-
.../org/apache/storm/starter/ReachTopology.java | 76 +--
.../starter/ResourceAwareExampleTopology.java | 113 ++--
.../apache/storm/starter/RollingTopWords.java | 111 ++-
.../apache/storm/starter/SingleJoinExample.java | 26 +-
.../storm/starter/SkewedRollingTopWords.java | 113 ++--
.../storm/starter/SlidingTupleTsTopology.java | 30 +-
.../storm/starter/SlidingWindowTopology.java | 62 +-
.../apache/storm/starter/StatefulTopology.java | 44 +-
.../starter/StatefulWindowingTopology.java | 46 +-
.../storm/starter/TransactionalGlobalCount.java | 65 +-
.../storm/starter/TransactionalWords.java | 102 ++-
.../apache/storm/starter/WordCountTopology.java | 115 ++--
.../storm/starter/WordCountTopologyNode.java | 61 +-
.../storm/starter/bolt/AbstractRankerBolt.java | 135 ++--
.../starter/bolt/IntermediateRankingsBolt.java | 61 +-
.../apache/storm/starter/bolt/PrinterBolt.java | 33 +-
.../storm/starter/bolt/RollingCountAggBolt.java | 92 ++-
.../storm/starter/bolt/RollingCountBolt.java | 169 +++--
.../storm/starter/bolt/SingleJoinBolt.java | 167 ++---
.../starter/bolt/SlidingWindowSumBolt.java | 48 +-
.../storm/starter/bolt/TotalRankingsBolt.java | 63 +-
.../storm/starter/spout/RandomIntegerSpout.java | 24 +-
.../spout/RandomNumberGeneratorSpout.java | 35 +-
.../starter/spout/RandomSentenceSpout.java | 120 ++--
.../storm/starter/streams/AggregateExample.java | 33 +-
.../storm/starter/streams/BranchExample.java | 45 +-
.../streams/GroupByKeyAndWindowExample.java | 62 +-
.../storm/starter/streams/JoinExample.java | 48 +-
.../starter/streams/StateQueryExample.java | 53 +-
.../starter/streams/StatefulWordCount.java | 59 +-
.../starter/streams/TypedTupleExample.java | 19 +-
.../starter/streams/WindowedWordCount.java | 70 +-
.../storm/starter/streams/WordCountToBolt.java | 51 +-
.../tools/NthLastModifiedTimeTracker.java | 75 +--
.../apache/storm/starter/tools/Rankable.java | 35 +-
.../starter/tools/RankableObjectWithFields.java | 224 +++----
.../apache/storm/starter/tools/Rankings.java | 238 ++++---
.../starter/tools/SlidingWindowCounter.java | 101 ++-
.../storm/starter/tools/SlotBasedCounter.java | 144 ++--
.../starter/trident/DebugMemoryMapState.java | 7 +-
.../TridentHBaseWindowingStoreTopology.java | 46 +-
.../starter/trident/TridentMapExample.java | 50 +-
.../trident/TridentMinMaxOfDevicesTopology.java | 85 ++-
.../TridentMinMaxOfVehiclesTopology.java | 83 ++-
.../storm/starter/trident/TridentReach.java | 95 ++-
.../TridentWindowingInmemoryStoreTopology.java | 43 +-
.../storm/starter/trident/TridentWordCount.java | 58 +-
.../bolt/IntermediateRankingsBoltTest.java | 244 ++++---
.../starter/bolt/RollingCountBoltTest.java | 178 +++--
.../starter/bolt/TotalRankingsBoltTest.java | 246 ++++---
.../tools/NthLastModifiedTimeTrackerTest.java | 158 +++--
.../tools/RankableObjectWithFieldsTest.java | 489 +++++++-------
.../storm/starter/tools/RankingsTest.java | 668 ++++++++++---------
.../starter/tools/SlidingWindowCounterTest.java | 175 +++--
.../starter/tools/SlotBasedCounterTest.java | 320 +++++----
67 files changed, 3613 insertions(+), 3869 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml
index d12062d..569daf5 100644
--- a/examples/storm-starter/pom.xml
+++ b/examples/storm-starter/pom.xml
@@ -249,7 +249,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>1538</maxAllowedViolations>
+ <maxAllowedViolations>263</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java
index cb45024..022f6ad 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/AnchoredWordCount.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.starter;
@@ -22,7 +16,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import java.util.UUID;
-
import org.apache.storm.Config;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -39,6 +32,27 @@ import org.apache.storm.utils.Utils;
public class AnchoredWordCount extends ConfigurableTopology {
+ protected int run(String[] args) throws Exception {
+ TopologyBuilder builder = new TopologyBuilder();
+
+ builder.setSpout("spout", new RandomSentenceSpout(), 4);
+
+ builder.setBolt("split", new SplitSentence(), 4).shuffleGrouping("spout");
+ builder.setBolt("count", new WordCount(), 4).fieldsGrouping("split", new Fields("word"));
+
+ Config conf = new Config();
+ conf.setMaxTaskParallelism(3);
+
+ String topologyName = "word-count";
+
+ conf.setNumWorkers(3);
+
+ if (args != null && args.length > 0) {
+ topologyName = args[0];
+ }
+ return submit(topologyName, conf, builder);
+ }
+
public static class RandomSentenceSpout extends BaseRichSpout {
SpoutOutputCollector collector;
Random random;
@@ -53,9 +67,11 @@ public class AnchoredWordCount extends ConfigurableTopology {
@Override
public void nextTuple() {
Utils.sleep(10);
- String[] sentences = new String[]{sentence("the cow jumped over the moon"), sentence("an apple a day keeps the doctor away"),
- sentence("four score and seven years ago"),
- sentence("snow white and the seven dwarfs"), sentence("i am at two with nature")};
+ String[] sentences = new String[]{
+ sentence("the cow jumped over the moon"), sentence("an apple a day keeps the doctor away"),
+ sentence("four score and seven years ago"),
+ sentence("snow white and the seven dwarfs"), sentence("i am at two with nature")
+ };
final String sentence = sentences[random.nextInt(sentences.length)];
this.collector.emit(new Values(sentence), UUID.randomUUID());
@@ -79,12 +95,11 @@ public class AnchoredWordCount extends ConfigurableTopology {
}
}
-
public static class SplitSentence extends BaseBasicBolt {
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
String sentence = tuple.getString(0);
- for (String word: sentence.split("\\s+")) {
+ for (String word : sentence.split("\\s+")) {
collector.emit(new Values(word, 1));
}
}
@@ -115,25 +130,4 @@ public class AnchoredWordCount extends ConfigurableTopology {
declarer.declare(new Fields("word", "count"));
}
}
-
- protected int run(String[] args) throws Exception {
- TopologyBuilder builder = new TopologyBuilder();
-
- builder.setSpout("spout", new RandomSentenceSpout(), 4);
-
- builder.setBolt("split", new SplitSentence(), 4).shuffleGrouping("spout");
- builder.setBolt("count", new WordCount(), 4).fieldsGrouping("split", new Fields("word"));
-
- Config conf = new Config();
- conf.setMaxTaskParallelism(3);
-
- String topologyName = "word-count";
-
- conf.setNumWorkers(3);
-
- if (args != null && args.length > 0) {
- topologyName = args[0];
- }
- return submit(topologyName, conf, builder);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java
index 325809c..83bad13 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/BasicDRPCTopology.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import org.apache.storm.Config;
@@ -35,19 +30,6 @@ import org.apache.storm.utils.DRPCClient;
* @see <a href="http://storm.apache.org/documentation/Distributed-RPC.html">Distributed RPC</a>
*/
public class BasicDRPCTopology {
- public static class ExclaimBolt extends BaseBasicBolt {
- @Override
- public void execute(Tuple tuple, BasicOutputCollector collector) {
- String input = tuple.getString(1);
- collector.emit(new Values(tuple.getValue(0), input + "!"));
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("id", "result"));
- }
- }
-
public static void main(String[] args) throws Exception {
LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation");
builder.addBolt(new ExclaimBolt(), 3);
@@ -56,12 +38,12 @@ public class BasicDRPCTopology {
String topoName = "DRPCExample";
if (args != null && args.length > 0) {
- topoName = args[0];
+ topoName = args[0];
}
conf.setNumWorkers(3);
StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createRemoteTopology());
-
+
if (args.length > 1) {
try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
for (int i = 1; i < args.length; i++) {
@@ -71,4 +53,17 @@ public class BasicDRPCTopology {
}
}
}
+
+ public static class ExclaimBolt extends BaseBasicBolt {
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ String input = tuple.getString(1);
+ collector.emit(new Values(tuple.getValue(0), input + "!"));
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("id", "result"));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java
index 2fc28d2..caa751a 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/BlobStoreAPIWordCountTopology.java
@@ -1,27 +1,36 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.StringTokenizer;
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.blobstore.AtomicOutputStream;
+import org.apache.storm.blobstore.BlobStoreAclHandler;
import org.apache.storm.blobstore.ClientBlobStore;
-
import org.apache.storm.generated.AccessControl;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
@@ -38,7 +47,6 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.topology.base.BaseRichSpout;
-import org.apache.storm.blobstore.BlobStoreAclHandler;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
@@ -46,26 +54,11 @@ import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.StringTokenizer;
-
public class BlobStoreAPIWordCountTopology {
+ private static final Logger LOG = LoggerFactory.getLogger(BlobStoreAPIWordCountTopology.class);
private static ClientBlobStore store; // Client API to invoke blob store API functionality
private static String key = "key";
private static String fileName = "blacklist.txt";
- private static final Logger LOG = LoggerFactory.getLogger(BlobStoreAPIWordCountTopology.class);
public static void prepare() {
Config conf = new Config();
@@ -73,114 +66,16 @@ public class BlobStoreAPIWordCountTopology {
store = Utils.getClientBlobStore(conf);
}
- // Spout implementation
- public static class RandomSentenceSpout extends BaseRichSpout {
- SpoutOutputCollector _collector;
-
- @Override
- public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
- _collector = collector;
- }
-
- @Override
- public void nextTuple() {
- Utils.sleep(100);
- _collector.emit(new Values(getRandomSentence()));
- }
-
- @Override
- public void ack(Object id) {
- }
-
- @Override
- public void fail(Object id) {
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("sentence"));
- }
-
- }
-
- // Bolt implementation
- public static class SplitSentence extends ShellBolt implements IRichBolt {
-
- public SplitSentence() {
- super("python", "splitsentence.py");
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("word"));
- }
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- return null;
- }
- }
-
- public static class FilterWords extends BaseBasicBolt {
- boolean poll = false;
- long pollTime;
- Set<String> wordSet;
- @Override
- public void execute(Tuple tuple, BasicOutputCollector collector) {
- String word = tuple.getString(0);
- // Thread Polling every 5 seconds to update the wordSet seconds which is
- // used in FilterWords bolt to filter the words
- try {
- if (!poll) {
- wordSet = parseFile(fileName);
- pollTime = System.currentTimeMillis();
- poll = true;
- } else {
- if ((System.currentTimeMillis() - pollTime) > 5000) {
- wordSet = parseFile(fileName);
- pollTime = System.currentTimeMillis();
- }
- }
- } catch (IOException exp) {
- throw new RuntimeException(exp);
- }
- if (wordSet !=null && !wordSet.contains(word)) {
- collector.emit(new Values(word));
- }
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("word"));
- }
- }
-
- public void buildAndLaunchWordCountTopology(String[] args) {
- TopologyBuilder builder = new TopologyBuilder();
- builder.setSpout("spout", new RandomSentenceSpout(), 5);
- builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
- builder.setBolt("filter", new FilterWords(), 6).shuffleGrouping("split");
-
- Config conf = new Config();
- conf.setDebug(true);
- try {
- conf.setNumWorkers(3);
- StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
- } catch (InvalidTopologyException | AuthorizationException | AlreadyAliveException exp) {
- throw new RuntimeException(exp);
- }
- }
-
// Equivalent create command on command line
// storm blobstore create --file blacklist.txt --acl o::rwa key
private static void createBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file)
- throws AuthorizationException, KeyAlreadyExistsException, IOException,KeyNotFoundException {
+ throws AuthorizationException, KeyAlreadyExistsException, IOException, KeyNotFoundException {
String stringBlobACL = "o::rwa";
AccessControl blobACL = BlobStoreAclHandler.parseAccessControl(stringBlobACL);
List<AccessControl> acls = new LinkedList<AccessControl>();
acls.add(blobACL); // more ACLs can be added here
SettableBlobMeta settableBlobMeta = new SettableBlobMeta(acls);
- AtomicOutputStream blobStream = clientBlobStore.createBlob(blobKey,settableBlobMeta);
+ AtomicOutputStream blobStream = clientBlobStore.createBlob(blobKey, settableBlobMeta);
blobStream.write(readFile(file).toString().getBytes());
blobStream.close();
}
@@ -188,15 +83,17 @@ public class BlobStoreAPIWordCountTopology {
// Equivalent update command on command line
// storm blobstore update --file blacklist.txt key
private static void updateBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file)
- throws KeyNotFoundException, AuthorizationException, IOException {
+ throws KeyNotFoundException, AuthorizationException, IOException {
AtomicOutputStream blobOutputStream = clientBlobStore.updateBlob(blobKey);
blobOutputStream.write(readFile(file).toString().getBytes());
blobOutputStream.close();
}
private static String getRandomSentence() {
- String[] sentences = new String[]{ "the cow jumped over the moon", "an apple a day keeps the doctor away",
- "four score and seven years ago", "snow white and the seven dwarfs", "i am at two with nature" };
+ String[] sentences = new String[]{
+ "the cow jumped over the moon", "an apple a day keeps the doctor away",
+ "four score and seven years ago", "snow white and the seven dwarfs", "i am at two with nature"
+ };
String sentence = sentences[new Random().nextInt(sentences.length)];
return sentence;
}
@@ -204,10 +101,12 @@ public class BlobStoreAPIWordCountTopology {
private static Set<String> getRandomWordSet() {
Set<String> randomWordSet = new HashSet<>();
Random random = new Random();
- String[] words = new String[]{ "cow", "jumped", "over", "the", "moon", "apple", "day", "doctor", "away",
- "four", "seven", "ago", "snow", "white", "seven", "dwarfs", "nature", "two" };
+ String[] words = new String[]{
+ "cow", "jumped", "over", "the", "moon", "apple", "day", "doctor", "away",
+ "four", "seven", "ago", "snow", "white", "seven", "dwarfs", "nature", "two"
+ };
// Choosing atmost 5 words to update the blacklist file for filtering
- for (int i=0; i<5; i++) {
+ for (int i = 0; i < 5; i++) {
randomWordSet.add(words[random.nextInt(words.length)]);
}
return randomWordSet;
@@ -259,11 +158,11 @@ public class BlobStoreAPIWordCountTopology {
}
// Writing random words to be blacklisted
- public static void writeToFile(File file, Set<String> content) throws IOException{
+ public static void writeToFile(File file, Set<String> content) throws IOException {
FileWriter fw = new FileWriter(file, false);
BufferedWriter bw = new BufferedWriter(fw);
Iterator<String> iter = content.iterator();
- while(iter.hasNext()) {
+ while (iter.hasNext()) {
bw.write(iter.next());
bw.write(System.lineSeparator());
}
@@ -286,7 +185,7 @@ public class BlobStoreAPIWordCountTopology {
wc.buildAndLaunchWordCountTopology(args);
// Updating file few times every 5 seconds
- for(int i=0; i<10; i++) {
+ for (int i = 0; i < 10; i++) {
updateBlobWithContent(key, store, updateFile(file));
Utils.sleep(5000);
}
@@ -296,6 +195,105 @@ public class BlobStoreAPIWordCountTopology {
throw new RuntimeException(exp);
}
}
+
+ public void buildAndLaunchWordCountTopology(String[] args) {
+ TopologyBuilder builder = new TopologyBuilder();
+ builder.setSpout("spout", new RandomSentenceSpout(), 5);
+ builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
+ builder.setBolt("filter", new FilterWords(), 6).shuffleGrouping("split");
+
+ Config conf = new Config();
+ conf.setDebug(true);
+ try {
+ conf.setNumWorkers(3);
+ StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
+ } catch (InvalidTopologyException | AuthorizationException | AlreadyAliveException exp) {
+ throw new RuntimeException(exp);
+ }
+ }
+
+ // Spout implementation
+ public static class RandomSentenceSpout extends BaseRichSpout {
+ SpoutOutputCollector _collector;
+
+ @Override
+ public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
+ _collector = collector;
+ }
+
+ @Override
+ public void nextTuple() {
+ Utils.sleep(100);
+ _collector.emit(new Values(getRandomSentence()));
+ }
+
+ @Override
+ public void ack(Object id) {
+ }
+
+ @Override
+ public void fail(Object id) {
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("sentence"));
+ }
+
+ }
+
+ // Bolt implementation
+ public static class SplitSentence extends ShellBolt implements IRichBolt {
+
+ public SplitSentence() {
+ super("python", "splitsentence.py");
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word"));
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return null;
+ }
+ }
+
+ public static class FilterWords extends BaseBasicBolt {
+ boolean poll = false;
+ long pollTime;
+ Set<String> wordSet;
+
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ String word = tuple.getString(0);
+ // Thread Polling every 5 seconds to update the wordSet seconds which is
+ // used in FilterWords bolt to filter the words
+ try {
+ if (!poll) {
+ wordSet = parseFile(fileName);
+ pollTime = System.currentTimeMillis();
+ poll = true;
+ } else {
+ if ((System.currentTimeMillis() - pollTime) > 5000) {
+ wordSet = parseFile(fileName);
+ pollTime = System.currentTimeMillis();
+ }
+ }
+ } catch (IOException exp) {
+ throw new RuntimeException(exp);
+ }
+ if (wordSet != null && !wordSet.contains(word)) {
+ collector.emit(new Values(word));
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word"));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java
index a691201..73f2067 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ExclamationTopology.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import java.util.Map;
-
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.testing.TestWordSpout;
@@ -35,48 +29,48 @@ import org.apache.storm.tuple.Values;
*/
public class ExclamationTopology extends ConfigurableTopology {
- public static class ExclamationBolt extends BaseRichBolt {
- OutputCollector _collector;
-
- @Override
- public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
- _collector = collector;
+ public static void main(String[] args) throws Exception {
+ ConfigurableTopology.start(new ExclamationTopology(), args);
}
- @Override
- public void execute(Tuple tuple) {
- _collector.emit(tuple, new Values(tuple.getString(0) + "!!!"));
- _collector.ack(tuple);
- }
+ protected int run(String[] args) {
+ TopologyBuilder builder = new TopologyBuilder();
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("word"));
- }
+ builder.setSpout("word", new TestWordSpout(), 10);
+ builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
+ builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
- }
+ conf.setDebug(true);
- public static void main(String[] args) throws Exception {
- ConfigurableTopology.start(new ExclamationTopology(), args);
- }
+ String topologyName = "test";
- protected int run(String[] args) {
- TopologyBuilder builder = new TopologyBuilder();
+ conf.setNumWorkers(3);
- builder.setSpout("word", new TestWordSpout(), 10);
- builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
- builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
+ if (args != null && args.length > 0) {
+ topologyName = args[0];
+ }
- conf.setDebug(true);
+ return submit(topologyName, conf, builder);
+ }
- String topologyName = "test";
+ public static class ExclamationBolt extends BaseRichBolt {
+ OutputCollector _collector;
- conf.setNumWorkers(3);
+ @Override
+ public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
+ _collector = collector;
+ }
- if (args != null && args.length > 0) {
- topologyName = args[0];
- }
+ @Override
+ public void execute(Tuple tuple) {
+ _collector.emit(tuple, new Values(tuple.getString(0) + "!!!"));
+ _collector.ack(tuple);
+ }
- return submit(topologyName, conf, builder);
- }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word"));
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java
index e171557..f881a86 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/FastWordCountTopology.java
@@ -1,25 +1,30 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
-import org.apache.storm.generated.*;
+import org.apache.storm.generated.ClusterSummary;
+import org.apache.storm.generated.ExecutorSummary;
+import org.apache.storm.generated.KillOptions;
+import org.apache.storm.generated.Nimbus;
+import org.apache.storm.generated.SpoutStats;
+import org.apache.storm.generated.TopologyInfo;
+import org.apache.storm.generated.TopologySummary;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
@@ -30,168 +35,165 @@ import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
-import org.apache.storm.utils.Utils;
import org.apache.storm.utils.NimbusClient;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
+import org.apache.storm.utils.Utils;
/**
* WordCount but the spout does not stop, and the bolts are implemented in
* java. This can show how fast the word count can run.
*/
public class FastWordCountTopology {
- public static class FastRandomSentenceSpout extends BaseRichSpout {
- SpoutOutputCollector _collector;
- Random _rand;
- private static final String[] CHOICES = {
- "marry had a little lamb whos fleese was white as snow",
- "and every where that marry went the lamb was sure to go",
- "one two three four five six seven eight nine ten",
- "this is a test of the emergency broadcast system this is only a test",
- "peter piper picked a peck of pickeled peppers"
- };
-
- @Override
- public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
- _collector = collector;
- _rand = ThreadLocalRandom.current();
+ public static void printMetrics(Nimbus.Iface client, String name) throws Exception {
+ ClusterSummary summary = client.getClusterInfo();
+ String id = null;
+ for (TopologySummary ts : summary.get_topologies()) {
+ if (name.equals(ts.get_name())) {
+ id = ts.get_id();
+ }
+ }
+ if (id == null) {
+ throw new Exception("Could not find a topology named " + name);
+ }
+ TopologyInfo info = client.getTopologyInfo(id);
+ int uptime = info.get_uptime_secs();
+ long acked = 0;
+ long failed = 0;
+ double weightedAvgTotal = 0.0;
+ for (ExecutorSummary exec : info.get_executors()) {
+ if ("spout".equals(exec.get_component_id())) {
+ SpoutStats stats = exec.get_stats().get_specific().get_spout();
+ Map<String, Long> failedMap = stats.get_failed().get(":all-time");
+ Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
+ Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
+ for (String key : ackedMap.keySet()) {
+ if (failedMap != null) {
+ Long tmp = failedMap.get(key);
+ if (tmp != null) {
+ failed += tmp;
+ }
+ }
+ long ackVal = ackedMap.get(key);
+ double latVal = avgLatMap.get(key) * ackVal;
+ acked += ackVal;
+ weightedAvgTotal += latVal;
+ }
+ }
+ }
+ double avgLatency = weightedAvgTotal / acked;
+ System.out.println("uptime: " + uptime + " acked: " + acked + " avgLatency: " + avgLatency + " acked/sec: " +
+ (((double) acked) / uptime + " failed: " + failed));
}
- @Override
- public void nextTuple() {
- String sentence = CHOICES[_rand.nextInt(CHOICES.length)];
- _collector.emit(new Values(sentence), sentence);
+ public static void kill(Nimbus.Iface client, String name) throws Exception {
+ KillOptions opts = new KillOptions();
+ opts.set_wait_secs(0);
+ client.killTopologyWithOpts(name, opts);
}
- @Override
- public void ack(Object id) {
- //Ignored
- }
+ public static void main(String[] args) throws Exception {
- @Override
- public void fail(Object id) {
- _collector.emit(new Values(id), id);
- }
+ TopologyBuilder builder = new TopologyBuilder();
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("sentence"));
- }
- }
-
- public static class SplitSentence extends BaseBasicBolt {
- @Override
- public void execute(Tuple tuple, BasicOutputCollector collector) {
- String sentence = tuple.getString(0);
- for (String word: sentence.split("\\s+")) {
- collector.emit(new Values(word, 1));
- }
- }
+ builder.setSpout("spout", new FastRandomSentenceSpout(), 4);
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("word", "count"));
- }
- }
-
- public static class WordCount extends BaseBasicBolt {
- Map<String, Integer> counts = new HashMap<String, Integer>();
-
- @Override
- public void execute(Tuple tuple, BasicOutputCollector collector) {
- String word = tuple.getString(0);
- Integer count = counts.get(word);
- if (count == null)
- count = 0;
- count++;
- counts.put(word, count);
- collector.emit(new Values(word, count));
- }
+ builder.setBolt("split", new SplitSentence(), 4).shuffleGrouping("spout");
+ builder.setBolt("count", new WordCount(), 4).fieldsGrouping("split", new Fields("word"));
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("word", "count"));
- }
- }
-
- public static void printMetrics(Nimbus.Iface client, String name) throws Exception {
- ClusterSummary summary = client.getClusterInfo();
- String id = null;
- for (TopologySummary ts: summary.get_topologies()) {
- if (name.equals(ts.get_name())) {
- id = ts.get_id();
- }
- }
- if (id == null) {
- throw new Exception("Could not find a topology named "+name);
- }
- TopologyInfo info = client.getTopologyInfo(id);
- int uptime = info.get_uptime_secs();
- long acked = 0;
- long failed = 0;
- double weightedAvgTotal = 0.0;
- for (ExecutorSummary exec: info.get_executors()) {
- if ("spout".equals(exec.get_component_id())) {
- SpoutStats stats = exec.get_stats().get_specific().get_spout();
- Map<String, Long> failedMap = stats.get_failed().get(":all-time");
- Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
- Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
- for (String key: ackedMap.keySet()) {
- if (failedMap != null) {
- Long tmp = failedMap.get(key);
- if (tmp != null) {
- failed += tmp;
- }
- }
- long ackVal = ackedMap.get(key);
- double latVal = avgLatMap.get(key) * ackVal;
- acked += ackVal;
- weightedAvgTotal += latVal;
+ Config conf = new Config();
+ conf.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class);
+
+ String name = "wc-test";
+ if (args != null && args.length > 0) {
+ name = args[0];
}
- }
- }
- double avgLatency = weightedAvgTotal/acked;
- System.out.println("uptime: "+uptime+" acked: "+acked+" avgLatency: "+avgLatency+" acked/sec: "+(((double)acked)/uptime+" failed: "+failed));
- }
- public static void kill(Nimbus.Iface client, String name) throws Exception {
- KillOptions opts = new KillOptions();
- opts.set_wait_secs(0);
- client.killTopologyWithOpts(name, opts);
- }
+ conf.setNumWorkers(1);
+ StormSubmitter.submitTopologyWithProgressBar(name, conf, builder.createTopology());
- public static void main(String[] args) throws Exception {
+ Map<String, Object> clusterConf = Utils.readStormConfig();
+ clusterConf.putAll(Utils.readCommandLineOpts());
+ Nimbus.Iface client = NimbusClient.getConfiguredClient(clusterConf).getClient();
- TopologyBuilder builder = new TopologyBuilder();
+ //Sleep for 5 mins
+ for (int i = 0; i < 10; i++) {
+ Thread.sleep(30 * 1000);
+ printMetrics(client, name);
+ }
+ kill(client, name);
+ }
+
+ public static class FastRandomSentenceSpout extends BaseRichSpout {
+ private static final String[] CHOICES = {
+ "marry had a little lamb whos fleese was white as snow",
+ "and every where that marry went the lamb was sure to go",
+ "one two three four five six seven eight nine ten",
+ "this is a test of the emergency broadcast system this is only a test",
+ "peter piper picked a peck of pickeled peppers"
+ };
+ SpoutOutputCollector _collector;
+ Random _rand;
+
+ @Override
+ public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
+ _collector = collector;
+ _rand = ThreadLocalRandom.current();
+ }
- builder.setSpout("spout", new FastRandomSentenceSpout(), 4);
+ @Override
+ public void nextTuple() {
+ String sentence = CHOICES[_rand.nextInt(CHOICES.length)];
+ _collector.emit(new Values(sentence), sentence);
+ }
- builder.setBolt("split", new SplitSentence(), 4).shuffleGrouping("spout");
- builder.setBolt("count", new WordCount(), 4).fieldsGrouping("split", new Fields("word"));
+ @Override
+ public void ack(Object id) {
+ //Ignored
+ }
- Config conf = new Config();
- conf.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class);
+ @Override
+ public void fail(Object id) {
+ _collector.emit(new Values(id), id);
+ }
- String name = "wc-test";
- if (args != null && args.length > 0) {
- name = args[0];
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("sentence"));
+ }
}
- conf.setNumWorkers(1);
- StormSubmitter.submitTopologyWithProgressBar(name, conf, builder.createTopology());
+ public static class SplitSentence extends BaseBasicBolt {
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ String sentence = tuple.getString(0);
+ for (String word : sentence.split("\\s+")) {
+ collector.emit(new Values(word, 1));
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word", "count"));
+ }
+ }
- Map<String, Object> clusterConf = Utils.readStormConfig();
- clusterConf.putAll(Utils.readCommandLineOpts());
- Nimbus.Iface client = NimbusClient.getConfiguredClient(clusterConf).getClient();
+ public static class WordCount extends BaseBasicBolt {
+ Map<String, Integer> counts = new HashMap<String, Integer>();
+
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ String word = tuple.getString(0);
+ Integer count = counts.get(word);
+ if (count == null) {
+ count = 0;
+ }
+ count++;
+ counts.put(word, count);
+ collector.emit(new Values(word, count));
+ }
- //Sleep for 5 mins
- for (int i = 0; i < 10; i++) {
- Thread.sleep(30 * 1000);
- printMetrics(client, name);
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word", "count"));
+ }
}
- kill(client, name);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java
index e4a41e2..5d314e0 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/InOrderDeliveryTest.java
@@ -1,25 +1,28 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
-import org.apache.storm.generated.*;
+import org.apache.storm.generated.ClusterSummary;
+import org.apache.storm.generated.ExecutorSummary;
+import org.apache.storm.generated.KillOptions;
+import org.apache.storm.generated.Nimbus;
+import org.apache.storm.generated.SpoutStats;
+import org.apache.storm.generated.TopologyInfo;
+import org.apache.storm.generated.TopologySummary;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
@@ -31,143 +34,141 @@ import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
-import org.apache.storm.utils.Utils;
import org.apache.storm.utils.NimbusClient;
-
-import java.util.HashMap;
-import java.util.Map;
+import org.apache.storm.utils.Utils;
public class InOrderDeliveryTest {
- public static class InOrderSpout extends BaseRichSpout {
- SpoutOutputCollector _collector;
- int _base = 0;
- int _i = 0;
-
- @Override
- public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
- _collector = collector;
- _base = context.getThisTaskIndex();
+ public static void printMetrics(Nimbus.Iface client, String name) throws Exception {
+ ClusterSummary summary = client.getClusterInfo();
+ String id = null;
+ for (TopologySummary ts : summary.get_topologies()) {
+ if (name.equals(ts.get_name())) {
+ id = ts.get_id();
+ }
+ }
+ if (id == null) {
+ throw new Exception("Could not find a topology named " + name);
+ }
+ TopologyInfo info = client.getTopologyInfo(id);
+ int uptime = info.get_uptime_secs();
+ long acked = 0;
+ long failed = 0;
+ double weightedAvgTotal = 0.0;
+ for (ExecutorSummary exec : info.get_executors()) {
+ if ("spout".equals(exec.get_component_id())) {
+ SpoutStats stats = exec.get_stats().get_specific().get_spout();
+ Map<String, Long> failedMap = stats.get_failed().get(":all-time");
+ Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
+ Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
+ for (String key : ackedMap.keySet()) {
+ if (failedMap != null) {
+ Long tmp = failedMap.get(key);
+ if (tmp != null) {
+ failed += tmp;
+ }
+ }
+ long ackVal = ackedMap.get(key);
+ double latVal = avgLatMap.get(key) * ackVal;
+ acked += ackVal;
+ weightedAvgTotal += latVal;
+ }
+ }
+ }
+ double avgLatency = weightedAvgTotal / acked;
+ System.out.println("uptime: " + uptime + " acked: " + acked + " avgLatency: " + avgLatency + " acked/sec: " +
+ (((double) acked) / uptime + " failed: " + failed));
}
- @Override
- public void nextTuple() {
- Values v = new Values(_base, _i);
- _collector.emit(v, "ACK");
- _i++;
+ public static void kill(Nimbus.Iface client, String name) throws Exception {
+ KillOptions opts = new KillOptions();
+ opts.set_wait_secs(0);
+ client.killTopologyWithOpts(name, opts);
}
- @Override
- public void ack(Object id) {
- //Ignored
- }
+ public static void main(String[] args) throws Exception {
- @Override
- public void fail(Object id) {
- //Ignored
- }
+ TopologyBuilder builder = new TopologyBuilder();
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("c1", "c2"));
- }
- }
-
- public static class Check extends BaseBasicBolt {
- Map<Integer, Integer> expected = new HashMap<Integer, Integer>();
-
- @Override
- public void execute(Tuple tuple, BasicOutputCollector collector) {
- Integer c1 = tuple.getInteger(0);
- Integer c2 = tuple.getInteger(1);
- Integer exp = expected.get(c1);
- if (exp == null) exp = 0;
- if (c2.intValue() != exp.intValue()) {
- System.out.println(c1+" "+c2+" != "+exp);
- throw new FailedException(c1+" "+c2+" != "+exp);
- }
- exp = c2 + 1;
- expected.put(c1, exp);
- }
+ builder.setSpout("spout", new InOrderSpout(), 8);
+ builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1"));
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- //Empty
- }
- }
-
- public static void printMetrics(Nimbus.Iface client, String name) throws Exception {
- ClusterSummary summary = client.getClusterInfo();
- String id = null;
- for (TopologySummary ts: summary.get_topologies()) {
- if (name.equals(ts.get_name())) {
- id = ts.get_id();
- }
- }
- if (id == null) {
- throw new Exception("Could not find a topology named "+name);
- }
- TopologyInfo info = client.getTopologyInfo(id);
- int uptime = info.get_uptime_secs();
- long acked = 0;
- long failed = 0;
- double weightedAvgTotal = 0.0;
- for (ExecutorSummary exec: info.get_executors()) {
- if ("spout".equals(exec.get_component_id())) {
- SpoutStats stats = exec.get_stats().get_specific().get_spout();
- Map<String, Long> failedMap = stats.get_failed().get(":all-time");
- Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
- Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
- for (String key: ackedMap.keySet()) {
- if (failedMap != null) {
- Long tmp = failedMap.get(key);
- if (tmp != null) {
- failed += tmp;
- }
- }
- long ackVal = ackedMap.get(key);
- double latVal = avgLatMap.get(key) * ackVal;
- acked += ackVal;
- weightedAvgTotal += latVal;
+ Config conf = new Config();
+ conf.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class);
+
+ String name = "in-order-test";
+ if (args != null && args.length > 0) {
+ name = args[0];
+ }
+
+ conf.setNumWorkers(1);
+ StormSubmitter.submitTopologyWithProgressBar(name, conf, builder.createTopology());
+
+ Map<String, Object> clusterConf = Utils.readStormConfig();
+ clusterConf.putAll(Utils.readCommandLineOpts());
+ Nimbus.Iface client = NimbusClient.getConfiguredClient(clusterConf).getClient();
+
+ //Sleep for 50 mins
+ for (int i = 0; i < 50; i++) {
+ Thread.sleep(30 * 1000);
+ printMetrics(client, name);
}
- }
+ kill(client, name);
}
- double avgLatency = weightedAvgTotal/acked;
- System.out.println("uptime: "+uptime+" acked: "+acked+" avgLatency: "+avgLatency+" acked/sec: "+(((double)acked)/uptime+" failed: "+failed));
- }
- public static void kill(Nimbus.Iface client, String name) throws Exception {
- KillOptions opts = new KillOptions();
- opts.set_wait_secs(0);
- client.killTopologyWithOpts(name, opts);
- }
+ public static class InOrderSpout extends BaseRichSpout {
+ SpoutOutputCollector _collector;
+ int _base = 0;
+ int _i = 0;
- public static void main(String[] args) throws Exception {
+ @Override
+ public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
+ _collector = collector;
+ _base = context.getThisTaskIndex();
+ }
- TopologyBuilder builder = new TopologyBuilder();
+ @Override
+ public void nextTuple() {
+ Values v = new Values(_base, _i);
+ _collector.emit(v, "ACK");
+ _i++;
+ }
- builder.setSpout("spout", new InOrderSpout(), 8);
- builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1"));
+ @Override
+ public void ack(Object id) {
+ //Ignored
+ }
- Config conf = new Config();
- conf.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class);
+ @Override
+ public void fail(Object id) {
+ //Ignored
+ }
- String name = "in-order-test";
- if (args != null && args.length > 0) {
- name = args[0];
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("c1", "c2"));
+ }
}
- conf.setNumWorkers(1);
- StormSubmitter.submitTopologyWithProgressBar(name, conf, builder.createTopology());
-
- Map<String, Object> clusterConf = Utils.readStormConfig();
- clusterConf.putAll(Utils.readCommandLineOpts());
- Nimbus.Iface client = NimbusClient.getConfiguredClient(clusterConf).getClient();
+ public static class Check extends BaseBasicBolt {
+ Map<Integer, Integer> expected = new HashMap<Integer, Integer>();
+
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ Integer c1 = tuple.getInteger(0);
+ Integer c2 = tuple.getInteger(1);
+ Integer exp = expected.get(c1);
+ if (exp == null) exp = 0;
+ if (c2.intValue() != exp.intValue()) {
+ System.out.println(c1 + " " + c2 + " != " + exp);
+ throw new FailedException(c1 + " " + c2 + " != " + exp);
+ }
+ exp = c2 + 1;
+ expected.put(c1, exp);
+ }
- //Sleep for 50 mins
- for (int i = 0; i < 50; i++) {
- Thread.sleep(30 * 1000);
- printMetrics(client, name);
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ //Empty
+ }
}
- kill(client, name);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java
index b71b64a..6fc3739 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/JoinBoltExample.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import java.util.concurrent.TimeUnit;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.bolt.JoinBolt;
@@ -33,8 +27,8 @@ import org.apache.storm.utils.NimbusClient;
public class JoinBoltExample {
public static void main(String[] args) throws Exception {
if (!NimbusClient.isLocalOverride()) {
- throw new IllegalStateException("This example only works in local mode. "
- + "Run with storm local not storm jar");
+ throw new IllegalStateException("This example only works in local mode. "
+ + "Run with storm local not storm jar");
}
FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));
@@ -45,15 +39,15 @@ public class JoinBoltExample {
// inner join of 'age' and 'gender' records on 'id' field
JoinBolt joiner = new JoinBolt("genderSpout", "id")
- .join("ageSpout", "id", "genderSpout")
- .select ("genderSpout:id,ageSpout:id,gender,age")
- .withTumblingWindow( new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS) );
+ .join("ageSpout", "id", "genderSpout")
+ .select("genderSpout:id,ageSpout:id,gender,age")
+ .withTumblingWindow(new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS));
builder.setBolt("joiner", joiner)
- .fieldsGrouping("genderSpout", new Fields("id"))
- .fieldsGrouping("ageSpout", new Fields("id")) ;
+ .fieldsGrouping("genderSpout", new Fields("id"))
+ .fieldsGrouping("ageSpout", new Fields("id"));
- builder.setBolt("printer", new PrinterBolt() ).shuffleGrouping("joiner");
+ builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("joiner");
Config conf = new Config();
StormSubmitter.submitTopologyWithProgressBar("join-example", conf, builder.createTopology());
@@ -74,8 +68,7 @@ public class JoinBoltExample {
String gender;
if (i % 2 == 0) {
gender = "male";
- }
- else {
+ } else {
gender = "female";
}
genderSpout.feed(new Values(i, gender));
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java
index 61b02db..94b1c38 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/LambdaTopology.java
@@ -1,30 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
+import java.io.Serializable;
+import java.util.UUID;
import org.apache.storm.Config;
import org.apache.storm.topology.ConfigurableTopology;
import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.tuple.Values;
-import java.io.Serializable;
-import java.util.UUID;
-
public class LambdaTopology extends ConfigurableTopology {
public static void main(String[] args) {
ConfigurableTopology.start(new LambdaTopology(), args);
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java
index 110d0be..2038399 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ManualDRPC.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import org.apache.storm.Config;
@@ -31,22 +26,6 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.DRPCClient;
public class ManualDRPC {
- public static class ExclamationBolt extends BaseBasicBolt {
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("result", "return-info"));
- }
-
- @Override
- public void execute(Tuple tuple, BasicOutputCollector collector) {
- String arg = tuple.getString(0);
- Object retInfo = tuple.getValue(1);
- collector.emit(new Values(arg + "!!!", retInfo));
- }
-
- }
-
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
@@ -62,4 +41,20 @@ public class ManualDRPC {
System.out.println(drpc.execute("exclamation", "bbb"));
}
}
+
+ public static class ExclamationBolt extends BaseBasicBolt {
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("result", "return-info"));
+ }
+
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ String arg = tuple.getString(0);
+ Object retInfo = tuple.getValue(1);
+ collector.emit(new Values(arg + "!!!", retInfo));
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java
index aeedc78..954e195 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/MultipleLoggerTopology.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.task.OutputCollector;
@@ -37,60 +31,60 @@ import org.slf4j.LoggerFactory;
* This is a basic example of a Storm topology.
*/
public class MultipleLoggerTopology {
- public static class ExclamationLoggingBolt extends BaseRichBolt {
- OutputCollector _collector;
- Logger _rootLogger = LoggerFactory.getLogger (Logger.ROOT_LOGGER_NAME);
- // ensure the loggers are configured in the worker.xml before
- // trying to use them here
- Logger _logger = LoggerFactory.getLogger ("com.myapp");
- Logger _subLogger = LoggerFactory.getLogger ("com.myapp.sub");
+ public static void main(String[] args) throws Exception {
+ TopologyBuilder builder = new TopologyBuilder();
- @Override
- public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
- _collector = collector;
- }
+ builder.setSpout("word", new TestWordSpout(), 10);
+ builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word");
+ builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1");
- @Override
- public void execute(Tuple tuple) {
- _rootLogger.debug ("root: This is a DEBUG message");
- _rootLogger.info ("root: This is an INFO message");
- _rootLogger.warn ("root: This is a WARN message");
- _rootLogger.error ("root: This is an ERROR message");
+ Config conf = new Config();
+ conf.setDebug(true);
+ String topoName = MultipleLoggerTopology.class.getName();
+ if (args != null && args.length > 0) {
+ topoName = args[0];
+ }
+ conf.setNumWorkers(2);
+ StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
+ }
- _logger.debug ("myapp: This is a DEBUG message");
- _logger.info ("myapp: This is an INFO message");
- _logger.warn ("myapp: This is a WARN message");
- _logger.error ("myapp: This is an ERROR message");
+ public static class ExclamationLoggingBolt extends BaseRichBolt {
+ OutputCollector _collector;
+ Logger _rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
+ // ensure the loggers are configured in the worker.xml before
+ // trying to use them here
+ Logger _logger = LoggerFactory.getLogger("com.myapp");
+ Logger _subLogger = LoggerFactory.getLogger("com.myapp.sub");
- _subLogger.debug ("myapp.sub: This is a DEBUG message");
- _subLogger.info ("myapp.sub: This is an INFO message");
- _subLogger.warn ("myapp.sub: This is a WARN message");
- _subLogger.error ("myapp.sub: This is an ERROR message");
+ @Override
+ public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
+ _collector = collector;
+ }
- _collector.emit(tuple, new Values(tuple.getString(0) + "!!!"));
- _collector.ack(tuple);
- }
+ @Override
+ public void execute(Tuple tuple) {
+ _rootLogger.debug("root: This is a DEBUG message");
+ _rootLogger.info("root: This is an INFO message");
+ _rootLogger.warn("root: This is a WARN message");
+ _rootLogger.error("root: This is an ERROR message");
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("word"));
- }
- }
+ _logger.debug("myapp: This is a DEBUG message");
+ _logger.info("myapp: This is an INFO message");
+ _logger.warn("myapp: This is a WARN message");
+ _logger.error("myapp: This is an ERROR message");
- public static void main(String[] args) throws Exception {
- TopologyBuilder builder = new TopologyBuilder();
+ _subLogger.debug("myapp.sub: This is a DEBUG message");
+ _subLogger.info("myapp.sub: This is an INFO message");
+ _subLogger.warn("myapp.sub: This is a WARN message");
+ _subLogger.error("myapp.sub: This is an ERROR message");
- builder.setSpout("word", new TestWordSpout(), 10);
- builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word");
- builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1");
+ _collector.emit(tuple, new Values(tuple.getString(0) + "!!!"));
+ _collector.ack(tuple);
+ }
- Config conf = new Config();
- conf.setDebug(true);
- String topoName = MultipleLoggerTopology.class.getName();
- if (args != null && args.length > 0) {
- topoName = args[0];
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word"));
+ }
}
- conf.setNumWorkers(2);
- StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java
index 566ab69..46f29a0 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/PersistentWindowingTopology.java
@@ -18,8 +18,6 @@
package org.apache.storm.starter;
-import static org.apache.storm.topology.base.BaseWindowedBolt.Duration;
-
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -40,18 +38,18 @@ import org.apache.storm.windowing.TupleWindow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.storm.topology.base.BaseWindowedBolt.Duration;
+
/**
- * An example that demonstrates the usage of {@link org.apache.storm.topology.IStatefulWindowedBolt} with window
- * persistence.
+ * An example that demonstrates the usage of {@link org.apache.storm.topology.IStatefulWindowedBolt} with window persistence.
* <p>
- * The framework automatically checkpoints the tuples in the window along with the bolt's state and restores the same
- * during restarts.
+ * The framework automatically checkpoints the tuples in the window along with the bolt's state and restores the same during restarts.
* </p>
*
* <p>
- * This topology uses 'redis' for state persistence, so you should also start a redis instance before deploying.
- * If you are running in local mode you can just start a redis server locally which will be used for storing the state. The default
- * RedisKeyValueStateProvider parameters can be overridden by setting {@link Config#TOPOLOGY_STATE_PROVIDER_CONFIG}, for e.g.
+ * This topology uses 'redis' for state persistence, so you should also start a redis instance before deploying. If you are running in local
+ * mode you can just start a redis server locally which will be used for storing the state. The default RedisKeyValueStateProvider
+ * parameters can be overridden by setting {@link Config#TOPOLOGY_STATE_PROVIDER_CONFIG}, for e.g.
* <pre>
* {
* "jedisPoolConfig": {
@@ -68,6 +66,47 @@ import org.slf4j.LoggerFactory;
public class PersistentWindowingTopology {
private static final Logger LOG = LoggerFactory.getLogger(PersistentWindowingTopology.class);
+ /**
+ * Create and deploy the topology.
+ *
+ * @param args args
+ * @throws Exception exception
+ */
+ public static void main(String[] args) throws Exception {
+ TopologyBuilder builder = new TopologyBuilder();
+
+ // generate random numbers
+ builder.setSpout("spout", new RandomIntegerSpout());
+
+ // emits sliding window and global averages
+ builder.setBolt("avgbolt", new AvgBolt()
+ .withWindow(new Duration(10, TimeUnit.SECONDS), new Duration(2, TimeUnit.SECONDS))
+ // persist the window in state
+ .withPersistence()
+ // max number of events to be cached in memory
+ .withMaxEventsInMemory(25000), 1)
+ .shuffleGrouping("spout");
+
+ // print the values to stdout
+ builder.setBolt("printer", (x, y) -> System.out.println(x.getValue(0)), 1).shuffleGrouping("avgbolt");
+
+ Config conf = new Config();
+ conf.setDebug(false);
+
+ // checkpoint the state every 5 seconds
+ conf.put(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL, 5000);
+
+ // use redis for state persistence
+ conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
+
+ String topoName = "test";
+ if (args != null && args.length > 0) {
+ topoName = args[0];
+ }
+ conf.setNumWorkers(1);
+ StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
+ }
+
// wrapper to hold global and window averages
private static class Averages {
private final double global;
@@ -131,46 +170,4 @@ public class PersistentWindowingTopology {
}
}
-
- /**
- * Create and deploy the topology.
- *
- * @param args args
- * @throws Exception exception
- */
- public static void main(String[] args) throws Exception {
- TopologyBuilder builder = new TopologyBuilder();
-
- // generate random numbers
- builder.setSpout("spout", new RandomIntegerSpout());
-
- // emits sliding window and global averages
- builder.setBolt("avgbolt", new AvgBolt()
- .withWindow(new Duration(10, TimeUnit.SECONDS), new Duration(2, TimeUnit.SECONDS))
- // persist the window in state
- .withPersistence()
- // max number of events to be cached in memory
- .withMaxEventsInMemory(25000), 1)
- .shuffleGrouping("spout");
-
- // print the values to stdout
- builder.setBolt("printer", (x, y) -> System.out.println(x.getValue(0)), 1).shuffleGrouping("avgbolt");
-
- Config conf = new Config();
- conf.setDebug(false);
-
- // checkpoint the state every 5 seconds
- conf.put(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL, 5000);
-
- // use redis for state persistence
- conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
-
- String topoName = "test";
- if (args != null && args.length > 0) {
- topoName = args[0];
- }
- conf.setNumWorkers(1);
- StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
- }
-
}
[36/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-redis
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-redis
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e3f5b138
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e3f5b138
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e3f5b138
Branch: refs/heads/master
Commit: e3f5b138eb33e0b67c98a3ba3d8b26d70c988537
Parents: 81ec15d
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 22:41:44 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 00:22:36 2018 -0400
----------------------------------------------------------------------
external/storm-redis/pom.xml | 2 +-
.../storm/redis/bolt/AbstractRedisBolt.java | 28 +-
.../storm/redis/bolt/RedisFilterBolt.java | 22 +-
.../storm/redis/bolt/RedisLookupBolt.java | 30 +--
.../apache/storm/redis/bolt/RedisStoreBolt.java | 19 +-
.../adapter/RedisCommandsAdapterJedis.java | 7 +-
.../RedisCommandsAdapterJedisCluster.java | 7 +-
.../redis/common/commands/RedisCommands.java | 9 +-
.../redis/common/config/JedisClusterConfig.java | 23 +-
.../redis/common/config/JedisPoolConfig.java | 44 ++--
.../common/container/JedisClusterContainer.java | 21 +-
.../JedisCommandsContainerBuilder.java | 26 +-
.../JedisCommandsInstanceContainer.java | 21 +-
.../redis/common/container/JedisContainer.java | 23 +-
.../common/container/RedisClusterContainer.java | 4 +-
.../RedisCommandsContainerBuilder.java | 10 +-
.../RedisCommandsInstanceContainer.java | 6 +-
.../redis/common/container/RedisContainer.java | 6 +-
.../common/mapper/RedisDataTypeDescription.java | 26 +-
.../redis/common/mapper/RedisFilterMapper.java | 19 +-
.../redis/common/mapper/RedisLookupMapper.java | 22 +-
.../storm/redis/common/mapper/RedisMapper.java | 19 +-
.../redis/common/mapper/RedisStoreMapper.java | 19 +-
.../storm/redis/common/mapper/TupleMapper.java | 22 +-
.../storm/redis/state/RedisKeyValueState.java | 62 ++---
.../redis/state/RedisKeyValueStateIterator.java | 22 +-
.../redis/state/RedisKeyValueStateProvider.java | 42 ++-
.../trident/state/AbstractRedisMapState.java | 37 ++-
.../state/AbstractRedisStateQuerier.java | 25 +-
.../state/AbstractRedisStateUpdater.java | 30 +--
.../storm/redis/trident/state/KeyFactory.java | 21 +-
.../storm/redis/trident/state/Options.java | 24 +-
.../trident/state/RedisClusterMapState.java | 213 ++++++++-------
.../redis/trident/state/RedisClusterState.java | 99 ++++---
.../trident/state/RedisClusterStateQuerier.java | 23 +-
.../trident/state/RedisClusterStateUpdater.java | 48 ++--
.../redis/trident/state/RedisMapState.java | 264 +++++++++----------
.../storm/redis/trident/state/RedisState.java | 92 +++----
.../redis/trident/state/RedisStateQuerier.java | 38 ++-
.../redis/trident/state/RedisStateUpdater.java | 48 ++--
.../state/RedisKeyValueStateIteratorTest.java | 69 +++--
.../state/RedisKeyValueStateProviderTest.java | 34 +--
.../redis/state/RedisKeyValueStateTest.java | 186 +++++++------
43 files changed, 797 insertions(+), 1015 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-redis/pom.xml b/external/storm-redis/pom.xml
index ca70344..5943496 100644
--- a/external/storm-redis/pom.xml
+++ b/external/storm-redis/pom.xml
@@ -92,7 +92,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>602</maxAllowedViolations>
+ <maxAllowedViolations>64</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/AbstractRedisBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/AbstractRedisBolt.java b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/AbstractRedisBolt.java
index f0f3fa3..06d59b6 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/AbstractRedisBolt.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/AbstractRedisBolt.java
@@ -1,37 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.bolt;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.base.BaseRichBolt;
+import java.util.Map;
import org.apache.storm.redis.common.config.JedisClusterConfig;
import org.apache.storm.redis.common.config.JedisPoolConfig;
import org.apache.storm.redis.common.container.JedisCommandsContainerBuilder;
import org.apache.storm.redis.common.container.JedisCommandsInstanceContainer;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.base.BaseTickTupleAwareRichBolt;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.utils.TupleUtils;
import redis.clients.jedis.JedisCommands;
-import java.util.Map;
-
/**
* AbstractRedisBolt class is for users to implement custom bolts which makes interaction with Redis.
* <p/>
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisFilterBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisFilterBolt.java b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisFilterBolt.java
index c7ba705..1b7ab20 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisFilterBolt.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisFilterBolt.java
@@ -1,22 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.bolt;
+import java.util.List;
import org.apache.storm.redis.common.config.JedisClusterConfig;
import org.apache.storm.redis.common.config.JedisPoolConfig;
import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
@@ -26,8 +22,6 @@ import org.apache.storm.tuple.Tuple;
import redis.clients.jedis.GeoCoordinate;
import redis.clients.jedis.JedisCommands;
-import java.util.List;
-
/**
* Basic bolt for querying from Redis and filters out if key/field doesn't exist.
* If key/field exists on Redis, this bolt just forwards input tuple to default stream.
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisLookupBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisLookupBolt.java b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisLookupBolt.java
index 0a4fde1..5311d85 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisLookupBolt.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisLookupBolt.java
@@ -1,33 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.bolt;
+import java.util.List;
+import org.apache.storm.redis.common.config.JedisClusterConfig;
+import org.apache.storm.redis.common.config.JedisPoolConfig;
+import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
+import org.apache.storm.redis.common.mapper.RedisLookupMapper;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
-import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
-import org.apache.storm.redis.common.mapper.RedisLookupMapper;
-import org.apache.storm.redis.common.config.JedisClusterConfig;
-import org.apache.storm.redis.common.config.JedisPoolConfig;
import redis.clients.jedis.JedisCommands;
-import java.util.List;
-
/**
* Basic bolt for querying from Redis and emits response as tuple.
* <p/>
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisStoreBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisStoreBolt.java b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisStoreBolt.java
index 7a910ce..8ca74e9 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisStoreBolt.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/bolt/RedisStoreBolt.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.bolt;
import org.apache.storm.redis.common.config.JedisClusterConfig;
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedis.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedis.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedis.java
index 66ac903..89d759d 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedis.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedis.java
@@ -18,15 +18,14 @@
package org.apache.storm.redis.common.adapter;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
import org.apache.storm.redis.common.commands.RedisCommands;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Map;
-
/**
* Adapter class to make Jedis instance play with BinaryRedisCommands interface.
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedisCluster.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedisCluster.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedisCluster.java
index be5cd64..680b3c9 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedisCluster.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/adapter/RedisCommandsAdapterJedisCluster.java
@@ -18,15 +18,14 @@
package org.apache.storm.redis.common.adapter;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
import org.apache.storm.redis.common.commands.RedisCommands;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Map;
-
/**
* Adapter class to make JedisCluster instance play with BinaryRedisCommands interface.
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/commands/RedisCommands.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/commands/RedisCommands.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/commands/RedisCommands.java
index dd01f20..172d2ab 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/commands/RedisCommands.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/commands/RedisCommands.java
@@ -18,16 +18,15 @@
package org.apache.storm.redis.common.commands;
+import java.util.Map;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
-import java.util.Map;
-
/**
* This interface represents Jedis methods exhaustively which are used on storm-redis.
*
- * This is a workaround since Jedis and JedisCluster doesn't implement same interface for binary type of methods,
- * and unify binary methods and string methods into one interface.
+ * This is a workaround since Jedis and JedisCluster doesn't implement same interface for binary type of methods, and unify binary methods
+ * and string methods into one interface.
*/
public interface RedisCommands {
// common
@@ -48,7 +47,7 @@ public interface RedisCommands {
Map<byte[], byte[]> hgetAll(byte[] key);
- Map<String,String> hgetAll(String key);
+ Map<String, String> hgetAll(String key);
String hmset(byte[] key, Map<byte[], byte[]> fieldValues);
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisClusterConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisClusterConfig.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisClusterConfig.java
index e7b444b..e9b355d 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisClusterConfig.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisClusterConfig.java
@@ -1,31 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.common.config;
import com.google.common.base.Preconditions;
-import redis.clients.jedis.HostAndPort;
-import redis.clients.jedis.Protocol;
-
import java.io.Serializable;
import java.net.InetSocketAddress;
import java.util.HashSet;
import java.util.Set;
+import redis.clients.jedis.HostAndPort;
+import redis.clients.jedis.Protocol;
/**
* Configuration for JedisCluster.
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisPoolConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisPoolConfig.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisPoolConfig.java
index ffd1244..f1025c9 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisPoolConfig.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/config/JedisPoolConfig.java
@@ -1,26 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.common.config;
-import redis.clients.jedis.Protocol;
-
import java.io.Serializable;
+import redis.clients.jedis.Protocol;
/**
* Configuration for JedisPool.
@@ -36,6 +29,7 @@ public class JedisPoolConfig implements Serializable {
// for serialization
public JedisPoolConfig() {
}
+
/**
* Constructor.
* <p/>
@@ -95,6 +89,17 @@ public class JedisPoolConfig implements Serializable {
return password;
}
+ @Override
+ public String toString() {
+ return "JedisPoolConfig{" +
+ "host='" + host + '\'' +
+ ", port=" + port +
+ ", timeout=" + timeout +
+ ", database=" + database +
+ ", password='" + password + '\'' +
+ '}';
+ }
+
/**
* Builder for initializing JedisPoolConfig.
*/
@@ -163,15 +168,4 @@ public class JedisPoolConfig implements Serializable {
return new JedisPoolConfig(host, port, timeout, password, database);
}
}
-
- @Override
- public String toString() {
- return "JedisPoolConfig{" +
- "host='" + host + '\'' +
- ", port=" + port +
- ", timeout=" + timeout +
- ", database=" + database +
- ", password='" + password + '\'' +
- '}';
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisClusterContainer.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisClusterContainer.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisClusterContainer.java
index 7d0baa7..9f1cb78 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisClusterContainer.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisClusterContainer.java
@@ -1,28 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.common.container;
+import java.io.IOException;
import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisCommands;
-import java.io.IOException;
-
/**
* Container for managing JedisCluster.
* <p/>
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainerBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainerBuilder.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainerBuilder.java
index d87553e..86f0f09 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainerBuilder.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsContainerBuilder.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.common.container;
@@ -38,7 +32,9 @@ public class JedisCommandsContainerBuilder {
* @return container for single Redis environment
*/
public static JedisCommandsInstanceContainer build(JedisPoolConfig config) {
- JedisPool jedisPool = new JedisPool(DEFAULT_POOL_CONFIG, config.getHost(), config.getPort(), config.getTimeout(), config.getPassword(), config.getDatabase());
+ JedisPool jedisPool =
+ new JedisPool(DEFAULT_POOL_CONFIG, config.getHost(), config.getPort(), config.getTimeout(), config.getPassword(),
+ config.getDatabase());
return new JedisContainer(jedisPool);
}
@@ -48,7 +44,9 @@ public class JedisCommandsContainerBuilder {
* @return container for Redis Cluster environment
*/
public static JedisCommandsInstanceContainer build(JedisClusterConfig config) {
- JedisCluster jedisCluster = new JedisCluster(config.getNodes(), config.getTimeout(), config.getTimeout(), config.getMaxRedirections(), config.getPassword(), DEFAULT_POOL_CONFIG);
+ JedisCluster jedisCluster =
+ new JedisCluster(config.getNodes(), config.getTimeout(), config.getTimeout(), config.getMaxRedirections(), config.getPassword(),
+ DEFAULT_POOL_CONFIG);
return new JedisClusterContainer(jedisCluster);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsInstanceContainer.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsInstanceContainer.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsInstanceContainer.java
index 17a61d6..8ca0716 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsInstanceContainer.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisCommandsInstanceContainer.java
@@ -1,26 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.common.container;
-import redis.clients.jedis.JedisCommands;
-
import java.io.Closeable;
+import redis.clients.jedis.JedisCommands;
/**
* Interfaces for containers which stores instances implementing JedisCommands.
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisContainer.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisContainer.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisContainer.java
index 14005f1..aa78480 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisContainer.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/JedisContainer.java
@@ -1,31 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.common.container;
+import java.io.Closeable;
+import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.JedisCommands;
import redis.clients.jedis.JedisPool;
-import java.io.Closeable;
-import java.io.IOException;
-
/**
* Container for managing Jedis instances.
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisClusterContainer.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisClusterContainer.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisClusterContainer.java
index d21ede8..512e950 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisClusterContainer.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisClusterContainer.java
@@ -18,12 +18,11 @@
package org.apache.storm.redis.common.container;
+import java.io.IOException;
import org.apache.storm.redis.common.adapter.RedisCommandsAdapterJedisCluster;
import org.apache.storm.redis.common.commands.RedisCommands;
import redis.clients.jedis.JedisCluster;
-import java.io.IOException;
-
/**
* Container for managing JedisCluster.
* <p/>
@@ -34,6 +33,7 @@ public class RedisClusterContainer implements RedisCommandsInstanceContainer {
/**
* Constructor
+ *
* @param jedisCluster JedisCluster instance
*/
public RedisClusterContainer(JedisCluster jedisCluster) {
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsContainerBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsContainerBuilder.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsContainerBuilder.java
index 856f88f..81201ef 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsContainerBuilder.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsContainerBuilder.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.redis.common.container;
import org.apache.storm.redis.common.config.JedisClusterConfig;
@@ -33,21 +34,26 @@ public class RedisCommandsContainerBuilder {
/**
* Builds container for single Redis environment.
+ *
* @param config configuration for JedisPool
* @return container for single Redis environment
*/
public static RedisCommandsInstanceContainer build(JedisPoolConfig config) {
- JedisPool jedisPool = new JedisPool(DEFAULT_POOL_CONFIG, config.getHost(), config.getPort(), config.getTimeout(), config.getPassword(), config.getDatabase());
+ JedisPool jedisPool =
+ new JedisPool(DEFAULT_POOL_CONFIG, config.getHost(), config.getPort(), config.getTimeout(), config.getPassword(),
+ config.getDatabase());
return new RedisContainer(jedisPool);
}
/**
* Builds container for Redis Cluster environment.
+ *
* @param config configuration for JedisCluster
* @return container for Redis Cluster environment
*/
public static RedisCommandsInstanceContainer build(JedisClusterConfig config) {
- JedisCluster jedisCluster = new JedisCluster(config.getNodes(), config.getTimeout(), config.getMaxRedirections(), DEFAULT_POOL_CONFIG);
+ JedisCluster jedisCluster =
+ new JedisCluster(config.getNodes(), config.getTimeout(), config.getMaxRedirections(), DEFAULT_POOL_CONFIG);
return new RedisClusterContainer(jedisCluster);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsInstanceContainer.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsInstanceContainer.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsInstanceContainer.java
index 04daf05..14b7c29 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsInstanceContainer.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisCommandsInstanceContainer.java
@@ -15,11 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.redis.common.container;
-import org.apache.storm.redis.common.commands.RedisCommands;
+package org.apache.storm.redis.common.container;
import java.io.Closeable;
+import org.apache.storm.redis.common.commands.RedisCommands;
/**
* Interfaces for containers which stores instances implementing RedisCommands.
@@ -27,12 +27,14 @@ import java.io.Closeable;
public interface RedisCommandsInstanceContainer extends Closeable {
/**
* Borrows instance from container.
+ *
* @return instance which implements RedisCommands
*/
RedisCommands getInstance();
/**
* Returns instance to container.
+ *
* @param redisCommands borrowed instance
*/
void returnInstance(RedisCommands redisCommands);
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisContainer.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisContainer.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisContainer.java
index e151346..1753bb8 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisContainer.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/container/RedisContainer.java
@@ -18,15 +18,14 @@
package org.apache.storm.redis.common.container;
+import java.io.Closeable;
+import java.io.IOException;
import org.apache.storm.redis.common.adapter.RedisCommandsAdapterJedis;
import org.apache.storm.redis.common.commands.RedisCommands;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.JedisPool;
-import java.io.Closeable;
-import java.io.IOException;
-
public class RedisContainer implements RedisCommandsInstanceContainer {
private static final Logger LOG = LoggerFactory.getLogger(JedisContainer.class);
@@ -34,6 +33,7 @@ public class RedisContainer implements RedisCommandsInstanceContainer {
/**
* Constructor
+ *
* @param jedisPool JedisPool which actually manages Jedis instances
*/
public RedisContainer(JedisPool jedisPool) {
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisDataTypeDescription.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisDataTypeDescription.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisDataTypeDescription.java
index 7e5c785..e3eac77 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisDataTypeDescription.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisDataTypeDescription.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.common.mapper;
import java.io.Serializable;
@@ -23,11 +18,8 @@ import java.io.Serializable;
* RedisDataTypeDescription defines data type and additional key if needed for lookup / store tuples.
*/
public class RedisDataTypeDescription implements Serializable {
- public enum RedisDataType { STRING, HASH, LIST, SET, SORTED_SET, HYPER_LOG_LOG, GEO }
-
private RedisDataType dataType;
private String additionalKey;
-
/**
* Constructor
* @param dataType data type
@@ -46,7 +38,7 @@ public class RedisDataTypeDescription implements Serializable {
this.additionalKey = additionalKey;
if (dataType == RedisDataType.HASH ||
- dataType == RedisDataType.SORTED_SET || dataType == RedisDataType.GEO) {
+ dataType == RedisDataType.SORTED_SET || dataType == RedisDataType.GEO) {
if (additionalKey == null) {
throw new IllegalArgumentException("Hash, Sorted Set and GEO should have additional key");
}
@@ -68,4 +60,6 @@ public class RedisDataTypeDescription implements Serializable {
public String getAdditionalKey() {
return additionalKey;
}
+
+ public enum RedisDataType {STRING, HASH, LIST, SET, SORTED_SET, HYPER_LOG_LOG, GEO}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisFilterMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisFilterMapper.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisFilterMapper.java
index b112046..f1d6506 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisFilterMapper.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisFilterMapper.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.common.mapper;
import org.apache.storm.topology.OutputFieldsDeclarer;
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisLookupMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisLookupMapper.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisLookupMapper.java
index 8c9ebf4..0e3b1c5 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisLookupMapper.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisLookupMapper.java
@@ -1,28 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.common.mapper;
+import java.util.List;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.ITuple;
import org.apache.storm.tuple.Values;
-import java.util.List;
-
/**
* RedisLookupMapper is for defining spec. which is used for querying value from Redis and converting response to tuple.
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisMapper.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisMapper.java
index b6a9559..b4ea07d 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisMapper.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisMapper.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.common.mapper;
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisStoreMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisStoreMapper.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisStoreMapper.java
index 4ab8b1f..f745d5d 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisStoreMapper.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/RedisStoreMapper.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.common.mapper;
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/TupleMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/TupleMapper.java b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/TupleMapper.java
index a2ab48b..ffe0d71 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/TupleMapper.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/common/mapper/TupleMapper.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.redis.common.mapper;
-import org.apache.storm.tuple.ITuple;
+package org.apache.storm.redis.common.mapper;
import java.io.Serializable;
+import org.apache.storm.tuple.ITuple;
/**
* TupleMapper defines how to extract key and value from tuple for Redis.
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueState.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueState.java
index 40b9ab4..95d0511 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueState.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueState.java
@@ -1,60 +1,52 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.state;
import com.google.common.collect.Maps;
import com.google.common.primitives.UnsignedBytes;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.storm.redis.common.commands.RedisCommands;
import org.apache.storm.redis.common.config.JedisClusterConfig;
+import org.apache.storm.redis.common.config.JedisPoolConfig;
import org.apache.storm.redis.common.container.RedisCommandsContainerBuilder;
import org.apache.storm.redis.common.container.RedisCommandsInstanceContainer;
import org.apache.storm.state.DefaultStateEncoder;
import org.apache.storm.state.DefaultStateSerializer;
import org.apache.storm.state.KeyValueState;
import org.apache.storm.state.Serializer;
-import org.apache.storm.redis.common.config.JedisPoolConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.util.SafeEncoder;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentNavigableMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-
/**
* A redis based implementation that persists the state in Redis.
*/
public class RedisKeyValueState<K, V> implements KeyValueState<K, V> {
public static final int ITERATOR_CHUNK_SIZE = 100;
-
+ public static final NavigableMap<byte[], byte[]> EMPTY_PENDING_COMMIT_MAP = Maps.unmodifiableNavigableMap(
+ new TreeMap<byte[], byte[]>(UnsignedBytes.lexicographicalComparator()));
private static final Logger LOG = LoggerFactory.getLogger(RedisKeyValueState.class);
private static final String COMMIT_TXID_KEY = "commit";
private static final String PREPARE_TXID_KEY = "prepare";
- public static final NavigableMap<byte[], byte[]> EMPTY_PENDING_COMMIT_MAP = Maps.unmodifiableNavigableMap(
- new TreeMap<byte[], byte[]>(UnsignedBytes.lexicographicalComparator()));
-
private final byte[] namespace;
private final byte[] prepareNamespace;
@@ -80,7 +72,8 @@ public class RedisKeyValueState<K, V> implements KeyValueState<K, V> {
this(namespace, RedisCommandsContainerBuilder.build(poolConfig), keySerializer, valueSerializer);
}
- public RedisKeyValueState(String namespace, JedisClusterConfig jedisClusterConfig, Serializer<K> keySerializer, Serializer<V> valueSerializer) {
+ public RedisKeyValueState(String namespace, JedisClusterConfig jedisClusterConfig, Serializer<K> keySerializer,
+ Serializer<V> valueSerializer) {
this(namespace, RedisCommandsContainerBuilder.build(jedisClusterConfig), keySerializer, valueSerializer);
}
@@ -181,8 +174,9 @@ public class RedisKeyValueState<K, V> implements KeyValueState<K, V> {
@Override
public Iterator<Map.Entry<K, V>> iterator() {
- return new RedisKeyValueStateIterator<K, V>(namespace, container, pendingPrepare.entrySet().iterator(), pendingCommit.entrySet().iterator(),
- ITERATOR_CHUNK_SIZE, encoder.getKeySerializer(), encoder.getValueSerializer());
+ return new RedisKeyValueStateIterator<K, V>(namespace, container, pendingPrepare.entrySet().iterator(),
+ pendingCommit.entrySet().iterator(),
+ ITERATOR_CHUNK_SIZE, encoder.getKeySerializer(), encoder.getValueSerializer());
}
@Override
@@ -196,7 +190,7 @@ public class RedisKeyValueState<K, V> implements KeyValueState<K, V> {
commands = container.getInstance();
if (commands.exists(prepareNamespace)) {
LOG.debug("Prepared txn already exists, will merge", txid);
- for (Map.Entry<byte[], byte[]> e: pendingCommit.entrySet()) {
+ for (Map.Entry<byte[], byte[]> e : pendingCommit.entrySet()) {
if (!currentPending.containsKey(e.getKey())) {
currentPending.put(e.getKey(), e.getValue());
}
@@ -226,7 +220,7 @@ public class RedisKeyValueState<K, V> implements KeyValueState<K, V> {
if (!pendingCommit.isEmpty()) {
List<byte[]> keysToDelete = new ArrayList<>();
Map<byte[], byte[]> keysToAdd = new HashMap<>();
- for(Map.Entry<byte[], byte[]> entry: pendingCommit.entrySet()) {
+ for (Map.Entry<byte[], byte[]> entry : pendingCommit.entrySet()) {
byte[] key = entry.getKey();
byte[] value = entry.getValue();
if (Arrays.equals(encoder.getTombstoneValue(), value)) {
@@ -306,7 +300,7 @@ public class RedisKeyValueState<K, V> implements KeyValueState<K, V> {
if (committedTxid != null) {
if (txid <= committedTxid) {
throw new RuntimeException("Invalid txid '" + txid + "' for prepare. Txid '" + committedTxid +
- "' is already committed");
+ "' is already committed");
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateIterator.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateIterator.java b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateIterator.java
index e959483..19d2b91 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateIterator.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateIterator.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.state;
@@ -22,13 +16,11 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.redis.common.commands.RedisCommands;
import org.apache.storm.redis.common.container.RedisCommandsInstanceContainer;
import org.apache.storm.state.BaseBinaryStateIterator;
import org.apache.storm.state.DefaultStateEncoder;
import org.apache.storm.state.Serializer;
-
import org.apache.storm.state.StateEncoder;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
@@ -79,7 +71,7 @@ public class RedisKeyValueStateIterator<K, V> extends BaseBinaryStateIterator<K,
@Override
protected boolean isEndOfDataFromStorage() {
return (cachedResultIterator == null || !cachedResultIterator.hasNext())
- && Arrays.equals(cursor, ScanParams.SCAN_POINTER_START_BINARY);
+ && Arrays.equals(cursor, ScanParams.SCAN_POINTER_START_BINARY);
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateProvider.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateProvider.java b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateProvider.java
index 01434ab..28201bc 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateProvider.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/state/RedisKeyValueStateProvider.java
@@ -1,25 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.state;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Collections;
+import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.redis.common.config.JedisClusterConfig;
import org.apache.storm.redis.common.config.JedisPoolConfig;
@@ -31,9 +28,6 @@ import org.apache.storm.task.TopologyContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collections;
-import java.util.Map;
-
/**
* Provides {@link RedisKeyValueState}
*/
@@ -75,10 +69,10 @@ public class RedisKeyValueStateProvider implements StateProvider {
if (jedisPoolConfig != null) {
return new RedisKeyValueState(namespace, jedisPoolConfig,
- getKeySerializer(topoConf, context, config), getValueSerializer(topoConf, context, config));
+ getKeySerializer(topoConf, context, config), getValueSerializer(topoConf, context, config));
} else {
return new RedisKeyValueState(namespace, jedisClusterConfig,
- getKeySerializer(topoConf, context, config), getValueSerializer(topoConf, context, config));
+ getKeySerializer(topoConf, context, config), getValueSerializer(topoConf, context, config));
}
}
@@ -131,13 +125,13 @@ public class RedisKeyValueStateProvider implements StateProvider {
@Override
public String toString() {
return "StateConfig{" +
- "keyClass='" + keyClass + '\'' +
- ", valueClass='" + valueClass + '\'' +
- ", keySerializerClass='" + keySerializerClass + '\'' +
- ", valueSerializerClass='" + valueSerializerClass + '\'' +
- ", jedisPoolConfig=" + jedisPoolConfig +
- ", jedisClusterConfig=" + jedisClusterConfig +
- '}';
+ "keyClass='" + keyClass + '\'' +
+ ", valueClass='" + valueClass + '\'' +
+ ", keySerializerClass='" + keySerializerClass + '\'' +
+ ", valueSerializerClass='" + valueSerializerClass + '\'' +
+ ", jedisPoolConfig=" + jedisPoolConfig +
+ ", jedisClusterConfig=" + jedisClusterConfig +
+ '}';
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java
index bf45fe4..6058b97 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisMapState.java
@@ -1,38 +1,31 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.trident.state;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
-import org.apache.storm.trident.state.JSONNonTransactionalSerializer;
-import org.apache.storm.trident.state.JSONOpaqueSerializer;
-import org.apache.storm.trident.state.JSONTransactionalSerializer;
-import org.apache.storm.trident.state.Serializer;
-import org.apache.storm.trident.state.StateType;
-import org.apache.storm.trident.state.map.IBackingMap;
-
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.storm.trident.state.JSONNonTransactionalSerializer;
+import org.apache.storm.trident.state.JSONOpaqueSerializer;
+import org.apache.storm.trident.state.JSONTransactionalSerializer;
+import org.apache.storm.trident.state.Serializer;
+import org.apache.storm.trident.state.StateType;
+import org.apache.storm.trident.state.map.IBackingMap;
/**
* AbstractRedisMapState is base class of any RedisMapState, which implements IBackingMap.
@@ -48,9 +41,9 @@ import java.util.Map;
*/
public abstract class AbstractRedisMapState<T> implements IBackingMap<T> {
public static final EnumMap<StateType, Serializer> DEFAULT_SERIALIZERS = Maps.newEnumMap(ImmutableMap.of(
- StateType.NON_TRANSACTIONAL, new JSONNonTransactionalSerializer(),
- StateType.TRANSACTIONAL, new JSONTransactionalSerializer(),
- StateType.OPAQUE, new JSONOpaqueSerializer()
+ StateType.NON_TRANSACTIONAL, new JSONNonTransactionalSerializer(),
+ StateType.TRANSACTIONAL, new JSONTransactionalSerializer(),
+ StateType.OPAQUE, new JSONOpaqueSerializer()
));
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateQuerier.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateQuerier.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateQuerier.java
index 9dbe4a8..0005989 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateQuerier.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateQuerier.java
@@ -1,33 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.trident.state;
-import org.apache.storm.tuple.Values;
import com.google.common.collect.Lists;
+import java.util.List;
import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
import org.apache.storm.redis.common.mapper.RedisLookupMapper;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseQueryFunction;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.tuple.TridentTuple;
-
-import java.util.List;
+import org.apache.storm.tuple.Values;
/**
* AbstractRedisStateQuerier is base class of any RedisStateQuerier, which implements BaseQueryFunction.
@@ -38,9 +31,9 @@ import java.util.List;
* @param <T> type of State
*/
public abstract class AbstractRedisStateQuerier<T extends State> extends BaseQueryFunction<T, List<Values>> {
- private final RedisLookupMapper lookupMapper;
protected final RedisDataTypeDescription.RedisDataType dataType;
protected final String additionalKey;
+ private final RedisLookupMapper lookupMapper;
/**
* Constructor
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateUpdater.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateUpdater.java
index 53c7b2b..84a32f7 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateUpdater.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/AbstractRedisStateUpdater.java
@@ -1,23 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.trident.state;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
import org.apache.storm.redis.common.mapper.RedisStoreMapper;
import org.apache.storm.trident.operation.TridentCollector;
@@ -25,10 +22,6 @@ import org.apache.storm.trident.state.BaseStateUpdater;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.tuple.TridentTuple;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
/**
* AbstractRedisStateUpdater is base class of any RedisStateUpdater, which implements BaseStateUpdater.
* <p/>
@@ -38,11 +31,10 @@ import java.util.Map;
* @param <T> type of State
*/
public abstract class AbstractRedisStateUpdater<T extends State> extends BaseStateUpdater<T> {
- private final RedisStoreMapper storeMapper;
-
- protected int expireIntervalSec = 0;
protected final RedisDataTypeDescription.RedisDataType dataType;
protected final String additionalKey;
+ private final RedisStoreMapper storeMapper;
+ protected int expireIntervalSec = 0;
/**
* Constructor
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/KeyFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/KeyFactory.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/KeyFactory.java
index 9d22aa5..cc8c06a 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/KeyFactory.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/KeyFactory.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.trident.state;
@@ -48,8 +42,9 @@ public interface KeyFactory extends Serializable {
*/
@Override
public String build(List<Object> key) {
- if (key.size() != 1)
+ if (key.size() != 1) {
throw new RuntimeException("Default KeyFactory does not support compound keys");
+ }
return (String) key.get(0);
}
[56/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-solr
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-solr
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/53adebcf
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/53adebcf
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/53adebcf
Branch: refs/heads/master
Commit: 53adebcfec5dec5e7a59f0a9108da734d3a3f01a
Parents: 95602b1
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 23:24:58 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 02:32:41 2018 -0400
----------------------------------------------------------------------
external/storm-solr/pom.xml | 2 +-
.../apache/storm/solr/bolt/SolrUpdateBolt.java | 30 ++--
.../storm/solr/config/CountBasedCommit.java | 18 +--
.../storm/solr/config/SolrCommitStrategy.java | 18 +--
.../apache/storm/solr/config/SolrConfig.java | 25 ++--
.../storm/solr/mapper/SolrFieldsMapper.java | 137 +++++++++----------
.../storm/solr/mapper/SolrJsonMapper.java | 119 ++++++++--------
.../apache/storm/solr/mapper/SolrMapper.java | 22 ++-
.../storm/solr/mapper/SolrMapperException.java | 18 +--
.../org/apache/storm/solr/schema/CopyField.java | 32 ++---
.../org/apache/storm/solr/schema/Field.java | 32 ++---
.../org/apache/storm/solr/schema/FieldType.java | 43 +++---
.../org/apache/storm/solr/schema/Schema.java | 82 +++++------
.../storm/solr/schema/SolrFieldTypeFinder.java | 87 ++++++------
.../schema/builder/RestJsonSchemaBuilder.java | 21 +--
.../schema/builder/RestJsonSchemaBuilderV2.java | 49 +++----
.../solr/schema/builder/SchemaBuilder.java | 20 +--
.../apache/storm/solr/trident/SolrState.java | 24 ++--
.../storm/solr/trident/SolrStateFactory.java | 19 +--
.../apache/storm/solr/trident/SolrUpdater.java | 21 +--
20 files changed, 350 insertions(+), 469 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-solr/pom.xml b/external/storm-solr/pom.xml
index b156f21..b65de8f 100644
--- a/external/storm-solr/pom.xml
+++ b/external/storm-solr/pom.xml
@@ -125,7 +125,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>108</maxAllowedViolations>
+ <maxAllowedViolations>47</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java b/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
index e525d92..f6c7c60 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/bolt/SolrUpdateBolt.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.bolt;
@@ -22,7 +16,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
@@ -68,15 +61,16 @@ public class SolrUpdateBolt extends BaseTickTupleAwareRichBolt {
this.solrMapper = solrMapper;
this.commitStgy = commitStgy;
LOG.debug("Created {} with the following configuration: " +
- "[SolrConfig = {}], [SolrMapper = {}], [CommitStgy = {}]",
- this.getClass().getSimpleName(), solrConfig, solrMapper, commitStgy);
+ "[SolrConfig = {}], [SolrMapper = {}], [CommitStgy = {}]",
+ this.getClass().getSimpleName(), solrConfig, solrMapper, commitStgy);
}
@Override
public void prepare(Map<String, Object> topoConf, TopologyContext context, OutputCollector collector) {
this.collector = collector;
- if (solrConfig.isKerberosEnabled())
+ if (solrConfig.isKerberosEnabled()) {
HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
+ }
solrMapper.configure();
this.solrClient = new CloudSolrClient(solrConfig.getZkHostString());
this.toCommitTuples = new ArrayList<>(capacity());
@@ -85,8 +79,8 @@ public class SolrUpdateBolt extends BaseTickTupleAwareRichBolt {
private int capacity() {
final int defArrListCpcty = 10;
return (commitStgy instanceof CountBasedCommit) ?
- ((CountBasedCommit)commitStgy).getThreshold() :
- defArrListCpcty;
+ ((CountBasedCommit) commitStgy).getThreshold() :
+ defArrListCpcty;
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/config/CountBasedCommit.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/config/CountBasedCommit.java b/external/storm-solr/src/main/java/org/apache/storm/solr/config/CountBasedCommit.java
index 6bd1220..1a9c967 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/config/CountBasedCommit.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/config/CountBasedCommit.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.config;
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrCommitStrategy.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrCommitStrategy.java b/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrCommitStrategy.java
index e4abf99..c4a8c3b 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrCommitStrategy.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrCommitStrategy.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.config;
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java b/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
index ecb3630..bd05e9d 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/config/SolrConfig.java
@@ -1,26 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.config;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-
import java.io.Serializable;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
/**
* Class containing Solr configuration to be made available to Storm Solr bolts. Any configuration needed in
@@ -35,7 +28,7 @@ public class SolrConfig implements Serializable {
* @param zkHostString Zookeeper host string as defined in the {@link CloudSolrClient} constructor
* */
public SolrConfig(String zkHostString) {
- this(zkHostString, 0);
+ this(zkHostString, 0);
}
/**
@@ -54,7 +47,7 @@ public class SolrConfig implements Serializable {
public SolrConfig(String zkHostString, int tickTupleInterval, boolean enableKerberos) {
this.zkHostString = zkHostString;
this.tickTupleInterval = tickTupleInterval;
- this.enableKerberos = enableKerberos;
+ this.enableKerberos = enableKerberos;
}
public String getZkHostString() {
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrFieldsMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrFieldsMapper.java b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrFieldsMapper.java
index b1d15cc..1ff7aa3 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrFieldsMapper.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrFieldsMapper.java
@@ -1,28 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.mapper;
-import static org.apache.storm.solr.schema.SolrFieldTypeFinder.FieldTypeWrapper;
-
import java.util.LinkedList;
import java.util.List;
-
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -36,73 +27,24 @@ import org.apache.storm.tuple.ITuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.storm.solr.schema.SolrFieldTypeFinder.FieldTypeWrapper;
+
public class SolrFieldsMapper implements SolrMapper {
private static final Logger log = LoggerFactory.getLogger(SolrFieldsMapper.class);
private String collection;
private SolrFieldTypeFinder typeFinder;
private String multiValueFieldToken;
- public static class Builder {
- private String collection;
- private SolrFieldTypeFinder typeFinder;
- private String multiValueFieldToken = "|";
-
- /**
- * {@link SolrFieldsMapper} builder class.
- * @param schemaBuilder Solr {@link Schema} builder class
- * @param collection Name of the Solr collection to update using the SolrRequest
- */
- public Builder(SchemaBuilder schemaBuilder, String collection) {
- setTypeFinder(schemaBuilder);
- this.collection = collection;
- }
-
- /**
- * {@link SolrFieldsMapper} builder class.
- * @param schemaBuilder Solr {@link Schema} builder class
- * @param solrClient {@link SolrClient} implementation from where to extract the default Solr collection, if any defined.
- */
- public Builder(SchemaBuilder schemaBuilder, SolrClient solrClient) {
- setTypeFinder(schemaBuilder);
- }
-
- //TODO Handle the case where there may be no schema
- private void setTypeFinder(SchemaBuilder schemaBuilder) {
- typeFinder = new SolrFieldTypeFinder(schemaBuilder);
- }
-
- // Sets {@link SolrFieldsMapper} to use the default Solr collection if there is one defined
- private void setDefaultCollection(SolrClient solrClient) {
- String defaultCollection = null;
- if (solrClient instanceof CloudSolrClient) {
- defaultCollection = ((CloudSolrClient) solrClient).getDefaultCollection();
- }
- this.collection = defaultCollection;
- }
-
- /**
- * Sets the token that separates multivalue fields in tuples. The default token is |
- * */
- public Builder setMultiValueFieldToken(String multiValueFieldToken) {
- this.multiValueFieldToken = multiValueFieldToken;
- return this;
- }
-
- public SolrFieldsMapper build() {
- return new SolrFieldsMapper(this);
- }
- }
-
private SolrFieldsMapper(Builder builder) {
this.collection = builder.collection;
this.typeFinder = builder.typeFinder;
this.multiValueFieldToken = builder.multiValueFieldToken;
log.debug("Created {} with the following configuration: [{}] "
- + this.getClass().getSimpleName(), this.toString());
+ + this.getClass().getSimpleName(), this.toString());
}
@Override
- public void configure() {
+ public void configure() {
typeFinder.initialize();
}
@@ -178,9 +120,60 @@ public class SolrFieldsMapper implements SolrMapper {
@Override
public String toString() {
return "SolrFieldsMapper{" +
- "collection='" + collection + '\'' +
- ", typeFinder=" + typeFinder +
- ", multiValueFieldToken='" + multiValueFieldToken + '\'' +
- '}';
+ "collection='" + collection + '\'' +
+ ", typeFinder=" + typeFinder +
+ ", multiValueFieldToken='" + multiValueFieldToken + '\'' +
+ '}';
+ }
+
+ public static class Builder {
+ private String collection;
+ private SolrFieldTypeFinder typeFinder;
+ private String multiValueFieldToken = "|";
+
+ /**
+ * {@link SolrFieldsMapper} builder class.
+ * @param schemaBuilder Solr {@link Schema} builder class
+ * @param collection Name of the Solr collection to update using the SolrRequest
+ */
+ public Builder(SchemaBuilder schemaBuilder, String collection) {
+ setTypeFinder(schemaBuilder);
+ this.collection = collection;
+ }
+
+ /**
+ * {@link SolrFieldsMapper} builder class.
+ * @param schemaBuilder Solr {@link Schema} builder class
+ * @param solrClient {@link SolrClient} implementation from where to extract the default Solr collection, if any defined.
+ */
+ public Builder(SchemaBuilder schemaBuilder, SolrClient solrClient) {
+ setTypeFinder(schemaBuilder);
+ }
+
+ //TODO Handle the case where there may be no schema
+ private void setTypeFinder(SchemaBuilder schemaBuilder) {
+ typeFinder = new SolrFieldTypeFinder(schemaBuilder);
+ }
+
+ // Sets {@link SolrFieldsMapper} to use the default Solr collection if there is one defined
+ private void setDefaultCollection(SolrClient solrClient) {
+ String defaultCollection = null;
+ if (solrClient instanceof CloudSolrClient) {
+ defaultCollection = ((CloudSolrClient) solrClient).getDefaultCollection();
+ }
+ this.collection = defaultCollection;
+ }
+
+ /**
+ * Sets the token that separates multivalue fields in tuples. The default token is |
+ * */
+ public Builder setMultiValueFieldToken(String multiValueFieldToken) {
+ this.multiValueFieldToken = multiValueFieldToken;
+ return this;
+ }
+
+ public SolrFieldsMapper build() {
+ return new SolrFieldsMapper(this);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
index 14a0a39..2334a0e 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrJsonMapper.java
@@ -1,26 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.mapper;
-import java.util.List;
-
import com.google.gson.Gson;
+import java.util.List;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -39,56 +32,8 @@ public class SolrJsonMapper implements SolrMapper {
private final String jsonTupleField;
private final String collection;
- public static class Builder {
- private String jsonUpdateUrl = "/update/json/docs";
- private final String jsonTupleField;
- private final String collection;
-
- /**
- * {@link SolrJsonMapper} builder class.
- * @param collection Name of the Solr collection to update using the SolrRequest
- * @param jsonTupleField Name of the tuple field that contains the JSON object used to update the Solr index
- */
- public Builder(String collection, String jsonTupleField) {
- this.collection = collection;
- this.jsonTupleField = jsonTupleField;
- }
-
- /**
- * {@link SolrJsonMapper} builder class.
- * @param solrClient {@link SolrClient} implementation from where to extract the default Solr collection, if any defined.
- * @param jsonTupleField Name of the tuple field that contains the JSON object used to update the Solr index
- */
- public Builder(SolrClient solrClient, String jsonTupleField) {
- String defaultCollection = null;
- if (solrClient instanceof CloudSolrClient) {
- defaultCollection = ((CloudSolrClient) solrClient).getDefaultCollection();
- }
- this.collection = defaultCollection;
- this.jsonTupleField = jsonTupleField;
- }
-
- /**
- * Sets the URL endpoint where to send {@link SolrRequest} requests in JSON format. This URL is defined
- * by Solr and for the most part you will not have to call this method to to override this value.
- * Currently the default value is /update/json/docs.
- * This method is available to support future evolutions of the Solr API.
- *
- * @param jsonUpdateUrl URL endpoint to where send {@link SolrRequest}
- * @return {@link Builder object} used to build instances of {@link SolrJsonMapper}
- */
- public Builder setJsonUpdateUrl(String jsonUpdateUrl) {
- this.jsonUpdateUrl = jsonUpdateUrl;
- return this;
- }
-
- public SolrJsonMapper build() {
- return new SolrJsonMapper(this);
- }
- }
-
private SolrJsonMapper(Builder builder) {
- jsonTupleField = builder.jsonTupleField ;
+ jsonTupleField = builder.jsonTupleField;
collection = builder.collection;
jsonUpdateUrl = builder.jsonUpdateUrl;
}
@@ -157,4 +102,52 @@ public class SolrJsonMapper implements SolrMapper {
}
return json;
}
+
+ public static class Builder {
+ private final String jsonTupleField;
+ private final String collection;
+ private String jsonUpdateUrl = "/update/json/docs";
+
+ /**
+ * {@link SolrJsonMapper} builder class.
+ * @param collection Name of the Solr collection to update using the SolrRequest
+ * @param jsonTupleField Name of the tuple field that contains the JSON object used to update the Solr index
+ */
+ public Builder(String collection, String jsonTupleField) {
+ this.collection = collection;
+ this.jsonTupleField = jsonTupleField;
+ }
+
+ /**
+ * {@link SolrJsonMapper} builder class.
+ * @param solrClient {@link SolrClient} implementation from where to extract the default Solr collection, if any defined.
+ * @param jsonTupleField Name of the tuple field that contains the JSON object used to update the Solr index
+ */
+ public Builder(SolrClient solrClient, String jsonTupleField) {
+ String defaultCollection = null;
+ if (solrClient instanceof CloudSolrClient) {
+ defaultCollection = ((CloudSolrClient) solrClient).getDefaultCollection();
+ }
+ this.collection = defaultCollection;
+ this.jsonTupleField = jsonTupleField;
+ }
+
+ /**
+ * Sets the URL endpoint where to send {@link SolrRequest} requests in JSON format. This URL is defined
+ * by Solr and for the most part you will not have to call this method to to override this value.
+ * Currently the default value is /update/json/docs.
+ * This method is available to support future evolutions of the Solr API.
+ *
+ * @param jsonUpdateUrl URL endpoint to where send {@link SolrRequest}
+ * @return {@link Builder object} used to build instances of {@link SolrJsonMapper}
+ */
+ public Builder setJsonUpdateUrl(String jsonUpdateUrl) {
+ this.jsonUpdateUrl = jsonUpdateUrl;
+ return this;
+ }
+
+ public SolrJsonMapper build() {
+ return new SolrJsonMapper(this);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapper.java b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapper.java
index c122aa4..1a083a2 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapper.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapper.java
@@ -1,32 +1,28 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.mapper;
import java.io.Serializable;
import java.util.List;
-
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.storm.tuple.ITuple;
public interface SolrMapper extends Serializable {
default void configure() {}
+
String getCollection();
+
SolrRequest toSolrRequest(ITuple tuple) throws SolrMapperException;
+
SolrRequest toSolrRequest(List<? extends ITuple> tuples) throws SolrMapperException;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapperException.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapperException.java b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapperException.java
index 654ee21..be3a82a 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapperException.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/mapper/SolrMapperException.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.mapper;
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/schema/CopyField.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/CopyField.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/CopyField.java
index f07584a..2281c8f 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/CopyField.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/CopyField.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.schema;
@@ -28,14 +22,14 @@ public class CopyField implements Serializable {
return source;
}
- public String getDest() {
- return dest;
- }
-
public void setSource(String source) {
this.source = source;
}
+ public String getDest() {
+ return dest;
+ }
+
public void setDest(String dest) {
this.dest = dest;
}
@@ -43,8 +37,8 @@ public class CopyField implements Serializable {
@Override
public String toString() {
return "CopyField{" +
- "source='" + source + '\'' +
- ", dest='" + dest + '\'' +
- '}';
+ "source='" + source + '\'' +
+ ", dest='" + dest + '\'' +
+ '}';
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Field.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Field.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Field.java
index 99b14a2..43b0110 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Field.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Field.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.schema;
@@ -28,14 +22,14 @@ public class Field implements Serializable {
return name;
}
- public String getType() {
- return type;
- }
-
public void setName(String name) {
this.name = name;
}
+ public String getType() {
+ return type;
+ }
+
public void setType(String type) {
this.type = type;
}
@@ -43,8 +37,8 @@ public class Field implements Serializable {
@Override
public String toString() {
return "Field{" +
- "name='" + name + '\'' +
- ", type='" + type + '\'' +
- '}';
+ "name='" + name + '\'' +
+ ", type='" + type + '\'' +
+ '}';
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/schema/FieldType.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/FieldType.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/FieldType.java
index e5d60bf..240af4b 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/FieldType.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/FieldType.java
@@ -1,25 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.schema;
import com.google.gson.annotations.SerializedName;
-
import java.io.Serializable;
public class FieldType implements Serializable {
@@ -32,22 +25,22 @@ public class FieldType implements Serializable {
return name;
}
- public String getClazz() {
- return clazz;
- }
-
- public boolean isMultiValued() {
- return multiValued;
- }
-
public void setName(String name) {
this.name = name;
}
+ public String getClazz() {
+ return clazz;
+ }
+
public void setClazz(String clazz) {
this.clazz = clazz;
}
+ public boolean isMultiValued() {
+ return multiValued;
+ }
+
public void setMultiValued(boolean multiValued) {
this.multiValued = multiValued;
}
@@ -55,9 +48,9 @@ public class FieldType implements Serializable {
@Override
public String toString() {
return "FieldType{" +
- "name='" + name + '\'' +
- ", clazz='" + clazz + '\'' +
- ", multiValued=" + multiValued +
- '}';
+ "name='" + name + '\'' +
+ ", clazz='" + clazz + '\'' +
+ ", multiValued=" + multiValued +
+ '}';
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Schema.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Schema.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Schema.java
index d8071dd..d9cdbac 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Schema.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/Schema.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.schema;
@@ -40,54 +34,54 @@ public class Schema implements Serializable {
return name;
}
- public String getVersion() {
- return version;
- }
-
- public String getUniqueKey() {
- return uniqueKey;
- }
-
- public List<FieldType> getFieldTypes() {
- return Collections.unmodifiableList(fieldTypes);
- }
-
- public List<Field> getFields() {
- return Collections.unmodifiableList(fields);
- }
-
- public List<Field> getDynamicFields() {
- return Collections.unmodifiableList(dynamicFields);
- }
-
- public List<CopyField> getCopyFields() {
- return Collections.unmodifiableList(copyFields);
- }
-
public void setName(String name) {
this.name = name;
}
+ public String getVersion() {
+ return version;
+ }
+
public void setVersion(String version) {
this.version = version;
}
+ public String getUniqueKey() {
+ return uniqueKey;
+ }
+
public void setUniqueKey(String uniqueKey) {
this.uniqueKey = uniqueKey;
}
+ public List<FieldType> getFieldTypes() {
+ return Collections.unmodifiableList(fieldTypes);
+ }
+
public void setFieldTypes(List<FieldType> fieldTypes) {
this.fieldTypes = fieldTypes;
}
+ public List<Field> getFields() {
+ return Collections.unmodifiableList(fields);
+ }
+
public void setFields(List<Field> fields) {
this.fields = fields;
}
+ public List<Field> getDynamicFields() {
+ return Collections.unmodifiableList(dynamicFields);
+ }
+
public void setDynamicFields(List<Field> dynamicFields) {
this.dynamicFields = dynamicFields;
}
+ public List<CopyField> getCopyFields() {
+ return Collections.unmodifiableList(copyFields);
+ }
+
public void setCopyFields(List<CopyField> copyFields) {
this.copyFields = copyFields;
}
@@ -95,14 +89,14 @@ public class Schema implements Serializable {
@Override
public String toString() {
return "Schema{" +
- "name='" + name + '\'' +
- ", version='" + version + '\'' +
- ", uniqueKey='" + uniqueKey + '\'' +
- ", fieldTypes=" + fieldTypes +
- ", fields=" + fields +
- ", dynamicFields=" + dynamicFields +
- ", copyFields=" + copyFields +
- '}';
+ "name='" + name + '\'' +
+ ", version='" + version + '\'' +
+ ", uniqueKey='" + uniqueKey + '\'' +
+ ", fieldTypes=" + fieldTypes +
+ ", fields=" + fields +
+ ", dynamicFields=" + dynamicFields +
+ ", copyFields=" + copyFields +
+ '}';
}
// Wrapper class handy for the client code to use the JSON parser to build to use with JSON parser
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/schema/SolrFieldTypeFinder.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/SolrFieldTypeFinder.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/SolrFieldTypeFinder.java
index c99d612..aa6802e 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/SolrFieldTypeFinder.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/SolrFieldTypeFinder.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.schema;
@@ -26,7 +20,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-
import org.apache.storm.solr.schema.builder.SchemaBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,40 +30,11 @@ import org.slf4j.LoggerFactory;
*/
public class SolrFieldTypeFinder implements Serializable {
private static final Logger log = LoggerFactory.getLogger(SolrFieldTypeFinder.class);
- private SchemaBuilder schemaBuilder;
+ private SchemaBuilder schemaBuilder;
private Schema schema;
private Map<String, FieldTypeWrapper> fieldToWrapper;
/**
- * Class wrapping all the information for fields and types
- * */
- public static class FieldTypeWrapper implements Serializable {
- Field field;
- FieldType type;
-
- public FieldTypeWrapper(Field field, FieldType type) {
- this.field = field;
- this.type = type;
- }
-
- public Field getField() {
- return field;
- }
-
- public FieldType getType() {
- return type;
- }
-
- @Override
- public String toString() {
- return "FieldTypeWrapper{" +
- "field=" + field +
- ", type=" + type +
- '}';
- }
- }
-
- /**
* Initiates class containing all the information relating fields with their types.
* This information is parsed from the schema
* @param schemaBuilder schemaBuilder to build the information about fields and types
@@ -105,11 +69,11 @@ public class SolrFieldTypeFinder implements Serializable {
}
private void buildMapForFields(List<FieldType> fieldTypes, List<Field> fields) {
- for (Field field: fields) {
+ for (Field field : fields) {
String type = field.getType();
int idx = indexOf(fieldTypes, type); // idx - index of the type of this field in the FieldType list
if (idx != -1) {
- fieldToWrapper.put(field.getName(), new FieldTypeWrapper(field, fieldTypes.get(idx)));
+ fieldToWrapper.put(field.getName(), new FieldTypeWrapper(field, fieldTypes.get(idx)));
}
}
}
@@ -133,7 +97,7 @@ public class SolrFieldTypeFinder implements Serializable {
* */
public FieldTypeWrapper getFieldTypeWrapper(String fieldName) {
FieldTypeWrapper typeWrapper = fieldToWrapper.get(fieldName);
- // => field name does not match static field, test if it matches dynamic field
+ // => field name does not match static field, test if it matches dynamic field
if (typeWrapper == null) {
for (String pattern : fieldToWrapper.keySet()) {
if (matchesDynamicField(fieldName, pattern)) {
@@ -181,7 +145,7 @@ public class SolrFieldTypeFinder implements Serializable {
return true;
}
} else if (pattern.endsWith("*")) {
- if (fieldName.startsWith(pattern.substring(0, pattern.length()-1))) {
+ if (fieldName.startsWith(pattern.substring(0, pattern.length() - 1))) {
log.debug("Field [{}] MATCHES dynamic field {}", fieldName, pattern);
return true;
}
@@ -191,4 +155,33 @@ public class SolrFieldTypeFinder implements Serializable {
}
return false;
}
+
+ /**
+ * Class wrapping all the information for fields and types
+ * */
+ public static class FieldTypeWrapper implements Serializable {
+ Field field;
+ FieldType type;
+
+ public FieldTypeWrapper(Field field, FieldType type) {
+ this.field = field;
+ this.type = type;
+ }
+
+ public Field getField() {
+ return field;
+ }
+
+ public FieldType getType() {
+ return type;
+ }
+
+ @Override
+ public String toString() {
+ return "FieldTypeWrapper{" +
+ "field=" + field +
+ ", type=" + type +
+ '}';
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
index b9b8f33..405429c 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilder.java
@@ -1,28 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.schema.builder;
+import com.google.gson.Gson;
import java.io.IOException;
import java.net.URL;
import java.util.Scanner;
-
-import com.google.gson.Gson;
import org.apache.storm.solr.schema.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilderV2.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilderV2.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilderV2.java
index 8cd66f8..aff8550 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilderV2.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/RestJsonSchemaBuilderV2.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.schema.builder;
@@ -22,7 +16,6 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -58,7 +51,7 @@ public class RestJsonSchemaBuilderV2 implements SchemaBuilder {
try {
solrClient = new CloudSolrClient(solrConfig.getZkHostString());
SchemaRequest schemaRequest = new SchemaRequest();
- logger.debug("Downloading schema for collection: {}", collection );
+ logger.debug("Downloading schema for collection: {}", collection);
SchemaResponse schemaResponse = schemaRequest.process(solrClient, collection);
logger.debug("SchemaResponse Schema: {}", schemaResponse);
SchemaRepresentation schemaRepresentation = schemaResponse.getSchemaRepresentation();
@@ -74,20 +67,22 @@ public class RestJsonSchemaBuilderV2 implements SchemaBuilder {
logger.error("Error while getting schema for collection: {}", collection, e);
throw new IOException("Error while getting schema for collection :" + collection, e);
} finally {
- if (solrClient != null)
+ if (solrClient != null) {
solrClient.close();
+ }
}
}
private List<FieldType> getFieldTypes(SchemaRepresentation schemaRepresentation) {
List<FieldType> fieldTypes = new LinkedList<>();
- for (FieldTypeDefinition fd: schemaRepresentation.getFieldTypes()) {
+ for (FieldTypeDefinition fd : schemaRepresentation.getFieldTypes()) {
FieldType ft = new FieldType();
- ft.setName((String)fd.getAttributes().get("name"));
- ft.setClazz((String)fd.getAttributes().get("class"));
+ ft.setName((String) fd.getAttributes().get("name"));
+ ft.setClazz((String) fd.getAttributes().get("class"));
Object multiValued = fd.getAttributes().get("multiValued");
- if (multiValued != null)
+ if (multiValued != null) {
ft.setMultiValued((Boolean) multiValued);
+ }
fieldTypes.add(ft);
}
return fieldTypes;
@@ -95,10 +90,10 @@ public class RestJsonSchemaBuilderV2 implements SchemaBuilder {
private List<Field> getFields(List<Map<String, Object>> schemaFields) {
List<Field> fields = new LinkedList<>();
- for (Map<String, Object> map: schemaFields) {
- Field field = new Field();
- field.setName((String)map.get("name"));
- field.setType((String)map.get("type"));
+ for (Map<String, Object> map : schemaFields) {
+ Field field = new Field();
+ field.setName((String) map.get("name"));
+ field.setType((String) map.get("type"));
fields.add(field);
}
return fields;
@@ -106,10 +101,10 @@ public class RestJsonSchemaBuilderV2 implements SchemaBuilder {
private List<CopyField> getCopyFields(SchemaRepresentation schemaRepresentation) {
List<CopyField> copyFields = new LinkedList<>();
- for (Map<String, Object> map: schemaRepresentation.getCopyFields()) {
- CopyField cp = new CopyField();
- cp.setSource((String)map.get("source"));
- cp.setDest((String)map.get("dest"));
+ for (Map<String, Object> map : schemaRepresentation.getCopyFields()) {
+ CopyField cp = new CopyField();
+ cp.setSource((String) map.get("source"));
+ cp.setDest((String) map.get("dest"));
copyFields.add(cp);
}
return copyFields;
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/SchemaBuilder.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/SchemaBuilder.java b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/SchemaBuilder.java
index cebb3fe..3f77e3d 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/SchemaBuilder.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/schema/builder/SchemaBuilder.java
@@ -1,29 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.schema.builder;
import java.io.IOException;
import java.io.Serializable;
-
import org.apache.storm.solr.schema.Schema;
public interface SchemaBuilder extends Serializable {
void buildSchema() throws IOException;
+
Schema getSchema();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
index 4668c5f..14b5717 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrState.java
@@ -1,25 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.trident;
import java.util.List;
-
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -46,14 +39,15 @@ public class SolrState implements State {
}
protected void prepare() {
- if (solrConfig.isKerberosEnabled())
+ if (solrConfig.isKerberosEnabled()) {
HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
+ }
solrMapper.configure();
solrClient = new CloudSolrClient(solrConfig.getZkHostString());
}
@Override
- public void beginCommit(Long aLong){ }
+ public void beginCommit(Long aLong) { }
@Override
public void commit(Long aLong) { }
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrStateFactory.java b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrStateFactory.java
index 2e8997c..b0ae4a2 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrStateFactory.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrStateFactory.java
@@ -1,25 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.trident;
import java.util.Map;
-
import org.apache.storm.solr.config.SolrConfig;
import org.apache.storm.solr.mapper.SolrMapper;
import org.apache.storm.task.IMetricsContext;
http://git-wip-us.apache.org/repos/asf/storm/blob/53adebcf/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrUpdater.java b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrUpdater.java
index 01c1c27..4e3cecc 100644
--- a/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrUpdater.java
+++ b/external/storm-solr/src/main/java/org/apache/storm/solr/trident/SolrUpdater.java
@@ -1,30 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.solr.trident;
import java.util.List;
-
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseStateUpdater;
import org.apache.storm.trident.tuple.TridentTuple;
-public class SolrUpdater extends BaseStateUpdater<SolrState> {
+public class SolrUpdater extends BaseStateUpdater<SolrState> {
@Override
public void updateState(SolrState solrState, List<TridentTuple> tuples, TridentCollector collector) {
[52/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-kafka-monitor
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-kafka-monitor
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/224633d3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/224633d3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/224633d3
Branch: refs/heads/master
Commit: 224633d3ccbaa843c7f94c709d5cc573b1c59845
Parents: 5fc4e9f
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 23:21:46 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 02:32:40 2018 -0400
----------------------------------------------------------------------
external/storm-kafka-monitor/pom.xml | 2 +-
.../kafka/monitor/KafkaOffsetLagResult.java | 14 +-
.../storm/kafka/monitor/KafkaOffsetLagUtil.java | 203 +++++++++++--------
.../kafka/monitor/KafkaPartitionOffsetLag.java | 81 ++++----
.../kafka/monitor/NewKafkaSpoutOffsetQuery.java | 12 +-
.../kafka/monitor/OldKafkaSpoutOffsetQuery.java | 25 ++-
6 files changed, 188 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/224633d3/external/storm-kafka-monitor/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-kafka-monitor/pom.xml b/external/storm-kafka-monitor/pom.xml
index 9e9b0a1..711dbea 100644
--- a/external/storm-kafka-monitor/pom.xml
+++ b/external/storm-kafka-monitor/pom.xml
@@ -147,7 +147,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>178</maxAllowedViolations>
+ <maxAllowedViolations>87</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/224633d3/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagResult.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagResult.java b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagResult.java
index a6d898c..908141c 100644
--- a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagResult.java
+++ b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagResult.java
@@ -61,12 +61,12 @@ public class KafkaOffsetLagResult implements JSONAware {
@Override
public String toString() {
return "KafkaOffsetLagResult{" +
- "topic='" + topic + '\'' +
- ", partition=" + partition +
- ", consumerCommittedOffset=" + consumerCommittedOffset +
- ", logHeadOffset=" + logHeadOffset +
- ", lag=" + lag +
- '}';
+ "topic='" + topic + '\'' +
+ ", partition=" + partition +
+ ", consumerCommittedOffset=" + consumerCommittedOffset +
+ ", logHeadOffset=" + logHeadOffset +
+ ", lag=" + lag +
+ '}';
}
@Override
@@ -97,6 +97,6 @@ public class KafkaOffsetLagResult implements JSONAware {
@Override
public String toJSONString() {
return "{\"topic\":\"" + topic + "\",\"partition\":" + partition + ",\"consumerCommittedOffset\":" + consumerCommittedOffset + "," +
- "\"logHeadOffset\":" + logHeadOffset + ",\"lag\":" + lag + "}";
+ "\"logHeadOffset\":" + logHeadOffset + ",\"lag\":" + lag + "}";
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/224633d3/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtil.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtil.java b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtil.java
index aa1a728..ef65bcb 100644
--- a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtil.java
+++ b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaOffsetLagUtil.java
@@ -18,6 +18,17 @@
package org.apache.storm.kafka.monitor;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import kafka.api.OffsetRequest;
+import kafka.api.PartitionOffsetRequestInfo;
+import kafka.common.TopicAndPartition;
+import kafka.javaapi.OffsetResponse;
+import kafka.javaapi.consumer.SimpleConsumer;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
@@ -32,19 +43,6 @@ import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.json.simple.JSONValue;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import kafka.api.OffsetRequest;
-import kafka.api.PartitionOffsetRequestInfo;
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.OffsetResponse;
-import kafka.javaapi.consumer.SimpleConsumer;
-
/**
* Utility class for querying offset lag for kafka spout
*/
@@ -72,7 +70,7 @@ public class KafkaOffsetLagUtil {
private static final String OPTION_SECURITY_PROTOCOL_SHORT = "s";
private static final String OPTION_SECURITY_PROTOCOL_LONG = "security-protocol";
- public static void main (String args[]) {
+ public static void main(String args[]) {
try {
List<KafkaOffsetLagResult> results;
Options options = buildOptions();
@@ -83,25 +81,29 @@ public class KafkaOffsetLagUtil {
}
if (commandLine.hasOption(OPTION_OLD_CONSUMER_LONG)) {
OldKafkaSpoutOffsetQuery oldKafkaSpoutOffsetQuery;
- if (commandLine.hasOption(OPTION_GROUP_ID_LONG) || commandLine.hasOption(OPTION_BOOTSTRAP_BROKERS_LONG) || commandLine.hasOption(OPTION_SECURITY_PROTOCOL_LONG)) {
- printUsageAndExit(options, OPTION_GROUP_ID_LONG + " or " + OPTION_BOOTSTRAP_BROKERS_LONG + " or " + OPTION_SECURITY_PROTOCOL_LONG + " is " +
- "not accepted with option " + OPTION_OLD_CONSUMER_LONG);
+ if (commandLine.hasOption(OPTION_GROUP_ID_LONG) || commandLine.hasOption(OPTION_BOOTSTRAP_BROKERS_LONG) ||
+ commandLine.hasOption(OPTION_SECURITY_PROTOCOL_LONG)) {
+ printUsageAndExit(options, OPTION_GROUP_ID_LONG + " or " + OPTION_BOOTSTRAP_BROKERS_LONG + " or " +
+ OPTION_SECURITY_PROTOCOL_LONG + " is " +
+ "not accepted with option " + OPTION_OLD_CONSUMER_LONG);
}
if (!commandLine.hasOption(OPTION_ZK_SERVERS_LONG) || !commandLine.hasOption(OPTION_ZK_COMMITTED_NODE_LONG)) {
printUsageAndExit(options, OPTION_ZK_SERVERS_LONG + " and " + OPTION_ZK_COMMITTED_NODE_LONG + " are required with " +
- OPTION_OLD_CONSUMER_LONG);
+ OPTION_OLD_CONSUMER_LONG);
}
String[] topics = commandLine.getOptionValue(OPTION_TOPIC_LONG).split(",");
if (topics != null && topics.length > 1) {
- printUsageAndExit(options, "Multiple topics not supported with option " + OPTION_OLD_CONSUMER_LONG + ". Either a single topic or a " +
- "wildcard string for matching topics is supported");
+ printUsageAndExit(options, "Multiple topics not supported with option " + OPTION_OLD_CONSUMER_LONG +
+ ". Either a single topic or a " +
+ "wildcard string for matching topics is supported");
}
if (commandLine.hasOption(OPTION_ZK_BROKERS_ROOT_LONG)) {
if (commandLine.hasOption(OPTION_PARTITIONS_LONG) || commandLine.hasOption(OPTION_LEADERS_LONG)) {
printUsageAndExit(options, OPTION_PARTITIONS_LONG + " or " + OPTION_LEADERS_LONG + " is not accepted with " +
- OPTION_ZK_BROKERS_ROOT_LONG);
+ OPTION_ZK_BROKERS_ROOT_LONG);
}
- oldKafkaSpoutOffsetQuery = new OldKafkaSpoutOffsetQuery(commandLine.getOptionValue(OPTION_TOPIC_LONG), commandLine.getOptionValue
+ oldKafkaSpoutOffsetQuery =
+ new OldKafkaSpoutOffsetQuery(commandLine.getOptionValue(OPTION_TOPIC_LONG), commandLine.getOptionValue
(OPTION_ZK_SERVERS_LONG), commandLine.getOptionValue(OPTION_ZK_COMMITTED_NODE_LONG), commandLine.hasOption
(OPTION_TOPIC_WILDCARD_LONG), commandLine.getOptionValue(OPTION_ZK_BROKERS_ROOT_LONG));
} else {
@@ -109,34 +111,41 @@ public class KafkaOffsetLagUtil {
printUsageAndExit(options, OPTION_TOPIC_WILDCARD_LONG + " is not supported without " + OPTION_ZK_BROKERS_ROOT_LONG);
}
if (!commandLine.hasOption(OPTION_PARTITIONS_LONG) || !commandLine.hasOption(OPTION_LEADERS_LONG)) {
- printUsageAndExit(options, OPTION_PARTITIONS_LONG + " and " + OPTION_LEADERS_LONG + " are required if " + OPTION_ZK_BROKERS_ROOT_LONG +
- " is not provided");
+ printUsageAndExit(options, OPTION_PARTITIONS_LONG + " and " + OPTION_LEADERS_LONG + " are required if " +
+ OPTION_ZK_BROKERS_ROOT_LONG +
+ " is not provided");
}
String[] partitions = commandLine.getOptionValue(OPTION_PARTITIONS_LONG).split(",");
String[] leaders = commandLine.getOptionValue(OPTION_LEADERS_LONG).split(",");
if (partitions.length != leaders.length) {
printUsageAndExit(options, OPTION_PARTITIONS_LONG + " and " + OPTION_LEADERS_LONG + " need to be of same size");
}
- oldKafkaSpoutOffsetQuery = new OldKafkaSpoutOffsetQuery(commandLine.getOptionValue(OPTION_TOPIC_LONG), commandLine.getOptionValue
+ oldKafkaSpoutOffsetQuery =
+ new OldKafkaSpoutOffsetQuery(commandLine.getOptionValue(OPTION_TOPIC_LONG), commandLine.getOptionValue
(OPTION_ZK_SERVERS_LONG), commandLine.getOptionValue(OPTION_ZK_COMMITTED_NODE_LONG), commandLine.getOptionValue
(OPTION_PARTITIONS_LONG), commandLine.getOptionValue(OPTION_LEADERS_LONG));
}
results = getOffsetLags(oldKafkaSpoutOffsetQuery);
} else {
String securityProtocol = commandLine.getOptionValue(OPTION_SECURITY_PROTOCOL_LONG);
- String[] oldSpoutOptions = {OPTION_TOPIC_WILDCARD_LONG, OPTION_PARTITIONS_LONG, OPTION_LEADERS_LONG, OPTION_ZK_SERVERS_LONG,
- OPTION_ZK_COMMITTED_NODE_LONG, OPTION_ZK_BROKERS_ROOT_LONG};
- for (String oldOption: oldSpoutOptions) {
+ String[] oldSpoutOptions = {
+ OPTION_TOPIC_WILDCARD_LONG, OPTION_PARTITIONS_LONG, OPTION_LEADERS_LONG, OPTION_ZK_SERVERS_LONG,
+ OPTION_ZK_COMMITTED_NODE_LONG, OPTION_ZK_BROKERS_ROOT_LONG
+ };
+ for (String oldOption : oldSpoutOptions) {
if (commandLine.hasOption(oldOption)) {
printUsageAndExit(options, oldOption + " is not accepted without " + OPTION_OLD_CONSUMER_LONG);
}
}
if (!commandLine.hasOption(OPTION_GROUP_ID_LONG) || !commandLine.hasOption(OPTION_BOOTSTRAP_BROKERS_LONG)) {
- printUsageAndExit(options, OPTION_GROUP_ID_LONG + " and " + OPTION_BOOTSTRAP_BROKERS_LONG + " are required if " + OPTION_OLD_CONSUMER_LONG +
- " is not specified");
+ printUsageAndExit(options, OPTION_GROUP_ID_LONG + " and " + OPTION_BOOTSTRAP_BROKERS_LONG + " are required if " +
+ OPTION_OLD_CONSUMER_LONG +
+ " is not specified");
}
- NewKafkaSpoutOffsetQuery newKafkaSpoutOffsetQuery = new NewKafkaSpoutOffsetQuery(commandLine.getOptionValue(OPTION_TOPIC_LONG),
- commandLine.getOptionValue(OPTION_BOOTSTRAP_BROKERS_LONG), commandLine.getOptionValue(OPTION_GROUP_ID_LONG), securityProtocol);
+ NewKafkaSpoutOffsetQuery newKafkaSpoutOffsetQuery =
+ new NewKafkaSpoutOffsetQuery(commandLine.getOptionValue(OPTION_TOPIC_LONG),
+ commandLine.getOptionValue(OPTION_BOOTSTRAP_BROKERS_LONG),
+ commandLine.getOptionValue(OPTION_GROUP_ID_LONG), securityProtocol);
results = getOffsetLags(newKafkaSpoutOffsetQuery);
}
@@ -161,49 +170,56 @@ public class KafkaOffsetLagUtil {
}
topicResultKeyedByPartition.put(result.getPartition(),
- new KafkaPartitionOffsetLag(result.getConsumerCommittedOffset(), result.getLogHeadOffset()));
+ new KafkaPartitionOffsetLag(result.getConsumerCommittedOffset(), result.getLogHeadOffset()));
}
return resultKeyedByTopic;
}
- private static void printUsageAndExit (Options options, String message) {
+ private static void printUsageAndExit(Options options, String message) {
System.out.println(message);
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("storm-kafka-monitor ", options);
System.exit(1);
}
- private static Options buildOptions () {
+ private static Options buildOptions() {
Options options = new Options();
- options.addOption(OPTION_TOPIC_SHORT, OPTION_TOPIC_LONG, true, "REQUIRED Topics (comma separated list) for fetching log head and spout committed " +
- "offset");
+ options.addOption(OPTION_TOPIC_SHORT, OPTION_TOPIC_LONG, true,
+ "REQUIRED Topics (comma separated list) for fetching log head and spout committed " +
+ "offset");
options.addOption(OPTION_OLD_CONSUMER_SHORT, OPTION_OLD_CONSUMER_LONG, false, "Whether request is for old spout");
- options.addOption(OPTION_BOOTSTRAP_BROKERS_SHORT, OPTION_BOOTSTRAP_BROKERS_LONG, true, "Comma separated list of bootstrap broker hosts for new " +
- "consumer/spout e.g. hostname1:9092,hostname2:9092");
+ options.addOption(OPTION_BOOTSTRAP_BROKERS_SHORT, OPTION_BOOTSTRAP_BROKERS_LONG, true,
+ "Comma separated list of bootstrap broker hosts for new " +
+ "consumer/spout e.g. hostname1:9092,hostname2:9092");
options.addOption(OPTION_GROUP_ID_SHORT, OPTION_GROUP_ID_LONG, true, "Group id of consumer (applicable only for new kafka spout) ");
- options.addOption(OPTION_TOPIC_WILDCARD_SHORT, OPTION_TOPIC_WILDCARD_LONG, false, "Whether topic provided is a wildcard as supported by ZkHosts in " +
- "old spout");
+ options.addOption(OPTION_TOPIC_WILDCARD_SHORT, OPTION_TOPIC_WILDCARD_LONG, false,
+ "Whether topic provided is a wildcard as supported by ZkHosts in " +
+ "old spout");
options.addOption(OPTION_PARTITIONS_SHORT, OPTION_PARTITIONS_LONG, true, "Comma separated list of partitions corresponding to " +
- OPTION_LEADERS_LONG + " for old spout with StaticHosts");
+ OPTION_LEADERS_LONG + " for old spout with StaticHosts");
options.addOption(OPTION_LEADERS_SHORT, OPTION_LEADERS_LONG, true, "Comma separated list of broker leaders corresponding to " +
- OPTION_PARTITIONS_LONG + " for old spout with StaticHosts e.g. hostname1:9092,hostname2:9092");
- options.addOption(OPTION_ZK_SERVERS_SHORT, OPTION_ZK_SERVERS_LONG, true, "Comma separated list of zk servers for fetching spout committed offsets " +
- "and/or topic metadata for ZkHosts e.g hostname1:2181,hostname2:2181");
- options.addOption(OPTION_ZK_COMMITTED_NODE_SHORT, OPTION_ZK_COMMITTED_NODE_LONG, true, "Zk node prefix where old kafka spout stores the committed" +
- " offsets without the topic and partition nodes");
- options.addOption(OPTION_ZK_BROKERS_ROOT_SHORT, OPTION_ZK_BROKERS_ROOT_LONG, true, "Zk node prefix where kafka stores broker information e.g. " +
- "/brokers (applicable only for old kafka spout) ");
+ OPTION_PARTITIONS_LONG +
+ " for old spout with StaticHosts e.g. hostname1:9092," +
+ "hostname2:9092");
+ options.addOption(OPTION_ZK_SERVERS_SHORT, OPTION_ZK_SERVERS_LONG, true,
+ "Comma separated list of zk servers for fetching spout committed offsets " +
+ "and/or topic metadata for ZkHosts e.g hostname1:2181,hostname2:2181");
+ options.addOption(OPTION_ZK_COMMITTED_NODE_SHORT, OPTION_ZK_COMMITTED_NODE_LONG, true,
+ "Zk node prefix where old kafka spout stores the committed" +
+ " offsets without the topic and partition nodes");
+ options.addOption(OPTION_ZK_BROKERS_ROOT_SHORT, OPTION_ZK_BROKERS_ROOT_LONG, true,
+ "Zk node prefix where kafka stores broker information e.g. " +
+ "/brokers (applicable only for old kafka spout) ");
options.addOption(OPTION_SECURITY_PROTOCOL_SHORT, OPTION_SECURITY_PROTOCOL_LONG, true, "Security protocol to connect to kafka");
return options;
}
/**
- *
* @param newKafkaSpoutOffsetQuery represents the information needed to query kafka for log head and spout offsets
* @return log head offset, spout offset and lag for each partition
*/
- public static List<KafkaOffsetLagResult> getOffsetLags (NewKafkaSpoutOffsetQuery newKafkaSpoutOffsetQuery) {
+ public static List<KafkaOffsetLagResult> getOffsetLags(NewKafkaSpoutOffsetQuery newKafkaSpoutOffsetQuery) {
KafkaConsumer<String, String> consumer = null;
List<KafkaOffsetLagResult> result = new ArrayList<>();
try {
@@ -219,7 +235,7 @@ public class KafkaOffsetLagUtil {
}
List<TopicPartition> topicPartitionList = new ArrayList<>();
consumer = new KafkaConsumer<>(props);
- for (String topic: newKafkaSpoutOffsetQuery.getTopics().split(",")) {
+ for (String topic : newKafkaSpoutOffsetQuery.getTopics().split(",")) {
List<PartitionInfo> partitionInfoList = consumer.partitionsFor(topic);
if (partitionInfoList != null) {
for (PartitionInfo partitionInfo : partitionInfoList) {
@@ -232,7 +248,8 @@ public class KafkaOffsetLagUtil {
OffsetAndMetadata offsetAndMetadata = consumer.committed(topicPartition);
long committedOffset = offsetAndMetadata != null ? offsetAndMetadata.offset() : -1;
consumer.seekToEnd(toArrayList(topicPartition));
- result.add(new KafkaOffsetLagResult(topicPartition.topic(), topicPartition.partition(), committedOffset, consumer.position(topicPartition)));
+ result.add(new KafkaOffsetLagResult(topicPartition.topic(), topicPartition.partition(), committedOffset,
+ consumer.position(topicPartition)));
}
} finally {
if (consumer != null) {
@@ -243,22 +260,23 @@ public class KafkaOffsetLagUtil {
}
private static Collection<TopicPartition> toArrayList(final TopicPartition tp) {
- return new ArrayList<TopicPartition>(1){{add(tp);}};
+ return new ArrayList<TopicPartition>(1) {{
+ add(tp);
+ }};
}
/**
- *
* @param oldKafkaSpoutOffsetQuery represents the information needed to query kafka for log head and spout offsets
* @return log head offset, spout offset and lag for each partition
*/
- public static List<KafkaOffsetLagResult> getOffsetLags (OldKafkaSpoutOffsetQuery oldKafkaSpoutOffsetQuery) throws Exception {
+ public static List<KafkaOffsetLagResult> getOffsetLags(OldKafkaSpoutOffsetQuery oldKafkaSpoutOffsetQuery) throws Exception {
List<KafkaOffsetLagResult> result = new ArrayList<>();
Map<String, List<TopicPartition>> leaders = getLeadersAndTopicPartitions(oldKafkaSpoutOffsetQuery);
if (leaders != null) {
Map<String, Map<Integer, Long>> logHeadOffsets = getLogHeadOffsets(leaders);
Map<String, List<Integer>> topicPartitions = new HashMap<>();
- for (Map.Entry<String, List<TopicPartition>> entry: leaders.entrySet()) {
- for (TopicPartition topicPartition: entry.getValue()) {
+ for (Map.Entry<String, List<TopicPartition>> entry : leaders.entrySet()) {
+ for (TopicPartition topicPartition : entry.getValue()) {
if (!topicPartitions.containsKey(topicPartition.topic())) {
topicPartitions.put(topicPartition.topic(), new ArrayList<Integer>());
}
@@ -266,13 +284,15 @@ public class KafkaOffsetLagUtil {
}
}
Map<String, Map<Integer, Long>> oldConsumerOffsets = getOldConsumerOffsetsFromZk(topicPartitions, oldKafkaSpoutOffsetQuery);
- for (Map.Entry<String, Map<Integer, Long>> topicOffsets: logHeadOffsets.entrySet()) {
- for (Map.Entry<Integer, Long> partitionOffsets: topicOffsets.getValue().entrySet()) {
- Long consumerCommittedOffset = oldConsumerOffsets.get(topicOffsets.getKey()) != null ? (Long) oldConsumerOffsets.get(topicOffsets.getKey()).get
- (partitionOffsets.getKey()) : -1;
+ for (Map.Entry<String, Map<Integer, Long>> topicOffsets : logHeadOffsets.entrySet()) {
+ for (Map.Entry<Integer, Long> partitionOffsets : topicOffsets.getValue().entrySet()) {
+ Long consumerCommittedOffset =
+ oldConsumerOffsets.get(topicOffsets.getKey()) != null ? (Long) oldConsumerOffsets.get(topicOffsets.getKey()).get
+ (partitionOffsets.getKey()) : -1;
consumerCommittedOffset = (consumerCommittedOffset == null ? -1 : consumerCommittedOffset);
KafkaOffsetLagResult kafkaOffsetLagResult = new KafkaOffsetLagResult(topicOffsets.getKey(), partitionOffsets.getKey(),
- consumerCommittedOffset, partitionOffsets.getValue());
+ consumerCommittedOffset,
+ partitionOffsets.getValue());
result.add(kafkaOffsetLagResult);
}
}
@@ -280,7 +300,8 @@ public class KafkaOffsetLagUtil {
return result;
}
- private static Map<String, List<TopicPartition>> getLeadersAndTopicPartitions (OldKafkaSpoutOffsetQuery oldKafkaSpoutOffsetQuery) throws Exception {
+ private static Map<String, List<TopicPartition>> getLeadersAndTopicPartitions(OldKafkaSpoutOffsetQuery oldKafkaSpoutOffsetQuery) throws
+ Exception {
Map<String, List<TopicPartition>> result = new HashMap<>();
// this means that kafka spout was configured with StaticHosts hosts (leader for partition)
if (oldKafkaSpoutOffsetQuery.getPartitions() != null) {
@@ -301,7 +322,8 @@ public class KafkaOffsetLagUtil {
brokersZkNode += "/";
}
String topicsZkPath = brokersZkNode + "topics";
- curatorFramework = CuratorFrameworkFactory.newClient(oldKafkaSpoutOffsetQuery.getZkServers(), 20000, 15000, new RetryOneTime(1000));
+ curatorFramework =
+ CuratorFrameworkFactory.newClient(oldKafkaSpoutOffsetQuery.getZkServers(), 20000, 15000, new RetryOneTime(1000));
curatorFramework.start();
List<String> topics = new ArrayList<>();
if (oldKafkaSpoutOffsetQuery.isWildCardTopic()) {
@@ -314,7 +336,7 @@ public class KafkaOffsetLagUtil {
} else {
topics.add(oldKafkaSpoutOffsetQuery.getTopic());
}
- for (String topic: topics) {
+ for (String topic : topics) {
String partitionsPath = topicsZkPath + "/" + topic + "/partitions";
List<String> children = curatorFramework.getChildren().forPath(partitionsPath);
for (int i = 0; i < children.size(); ++i) {
@@ -341,27 +363,33 @@ public class KafkaOffsetLagUtil {
return result;
}
- private static Map<String, Map<Integer, Long>> getLogHeadOffsets (Map<String, List<TopicPartition>> leadersAndTopicPartitions) {
+ private static Map<String, Map<Integer, Long>> getLogHeadOffsets(Map<String, List<TopicPartition>> leadersAndTopicPartitions) {
Map<String, Map<Integer, Long>> result = new HashMap<>();
if (leadersAndTopicPartitions != null) {
PartitionOffsetRequestInfo partitionOffsetRequestInfo = new PartitionOffsetRequestInfo(OffsetRequest.LatestTime(), 1);
- SimpleConsumer simpleConsumer = null;
- for (Map.Entry<String, List<TopicPartition>> leader: leadersAndTopicPartitions.entrySet()) {
+ SimpleConsumer simpleConsumer = null;
+ for (Map.Entry<String, List<TopicPartition>> leader : leadersAndTopicPartitions.entrySet()) {
try {
- simpleConsumer = new SimpleConsumer(leader.getKey().split(":")[0], Integer.parseInt(leader.getKey().split(":")[1]), 10000, 64 *
- 1024, "LogHeadOffsetRequest");
- Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
+ simpleConsumer =
+ new SimpleConsumer(leader.getKey().split(":")[0], Integer.parseInt(leader.getKey().split(":")[1]), 10000, 64 *
+ 1024,
+ "LogHeadOffsetRequest");
+ Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
+ new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
for (TopicPartition topicPartition : leader.getValue()) {
- requestInfo.put(new TopicAndPartition(topicPartition.topic(), topicPartition.partition()), partitionOffsetRequestInfo);
+ requestInfo
+ .put(new TopicAndPartition(topicPartition.topic(), topicPartition.partition()), partitionOffsetRequestInfo);
if (!result.containsKey(topicPartition.topic())) {
result.put(topicPartition.topic(), new HashMap<Integer, Long>());
}
}
- kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(),
- "LogHeadOffsetRequest");
+ kafka.javaapi.OffsetRequest request =
+ new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(),
+ "LogHeadOffsetRequest");
OffsetResponse response = simpleConsumer.getOffsetsBefore(request);
for (TopicPartition topicPartition : leader.getValue()) {
- result.get(topicPartition.topic()).put(topicPartition.partition(), response.offsets(topicPartition.topic(), topicPartition.partition())[0]);
+ result.get(topicPartition.topic())
+ .put(topicPartition.partition(), response.offsets(topicPartition.topic(), topicPartition.partition())[0]);
}
} finally {
if (simpleConsumer != null) {
@@ -373,29 +401,34 @@ public class KafkaOffsetLagUtil {
return result;
}
- private static Map<String, Map<Integer, Long>> getOldConsumerOffsetsFromZk (Map<String, List<Integer>> topicPartitions, OldKafkaSpoutOffsetQuery
- oldKafkaSpoutOffsetQuery) throws Exception {
+ private static Map<String, Map<Integer, Long>> getOldConsumerOffsetsFromZk(Map<String, List<Integer>> topicPartitions,
+ OldKafkaSpoutOffsetQuery
+ oldKafkaSpoutOffsetQuery) throws Exception {
Map<String, Map<Integer, Long>> result = new HashMap<>();
- CuratorFramework curatorFramework = CuratorFrameworkFactory.newClient(oldKafkaSpoutOffsetQuery.getZkServers(), 20000, 15000, new RetryOneTime(1000));
+ CuratorFramework curatorFramework =
+ CuratorFrameworkFactory.newClient(oldKafkaSpoutOffsetQuery.getZkServers(), 20000, 15000, new RetryOneTime(1000));
curatorFramework.start();
String partitionPrefix = "partition_";
String zkPath = oldKafkaSpoutOffsetQuery.getZkPath();
if (zkPath.endsWith("/")) {
- zkPath = zkPath.substring(0, zkPath.length()-1);
+ zkPath = zkPath.substring(0, zkPath.length() - 1);
}
if (curatorFramework.checkExists().forPath(zkPath) == null) {
- throw new IllegalArgumentException(OPTION_ZK_COMMITTED_NODE_LONG+" '"+zkPath+"' dose not exists.");
+ throw new IllegalArgumentException(OPTION_ZK_COMMITTED_NODE_LONG + " '" + zkPath + "' dose not exists.");
}
byte[] zkData;
try {
if (topicPartitions != null) {
- for (Map.Entry<String, List<Integer>> topicEntry: topicPartitions.entrySet()) {
+ for (Map.Entry<String, List<Integer>> topicEntry : topicPartitions.entrySet()) {
Map<Integer, Long> partitionOffsets = new HashMap<>();
- for (Integer partition: topicEntry.getValue()) {
- String path = zkPath + "/" + (oldKafkaSpoutOffsetQuery.isWildCardTopic() ? topicEntry.getKey() + "/" : "") + partitionPrefix + partition;
+ for (Integer partition : topicEntry.getValue()) {
+ String path =
+ zkPath + "/" + (oldKafkaSpoutOffsetQuery.isWildCardTopic() ? topicEntry.getKey() + "/" : "") + partitionPrefix +
+ partition;
if (curatorFramework.checkExists().forPath(path) != null) {
zkData = curatorFramework.getData().forPath(path);
- Map<Object, Object> offsetData = (Map<Object, Object>) JSONValue.parseWithException(new String(zkData, "UTF-8"));
+ Map<Object, Object> offsetData =
+ (Map<Object, Object>) JSONValue.parseWithException(new String(zkData, "UTF-8"));
partitionOffsets.put(partition, (Long) offsetData.get("offset"));
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/224633d3/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaPartitionOffsetLag.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaPartitionOffsetLag.java b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaPartitionOffsetLag.java
index 6faf266..a47cef2 100644
--- a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaPartitionOffsetLag.java
+++ b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/KafkaPartitionOffsetLag.java
@@ -15,57 +15,58 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.kafka.monitor;
public class KafkaPartitionOffsetLag {
- private long consumerCommittedOffset;
- private long logHeadOffset;
- private long lag;
+ private long consumerCommittedOffset;
+ private long logHeadOffset;
+ private long lag;
- public KafkaPartitionOffsetLag(long consumerCommittedOffset, long logHeadOffset) {
- this.consumerCommittedOffset = consumerCommittedOffset;
- this.logHeadOffset = logHeadOffset;
- this.lag = logHeadOffset - consumerCommittedOffset;
- }
+ public KafkaPartitionOffsetLag(long consumerCommittedOffset, long logHeadOffset) {
+ this.consumerCommittedOffset = consumerCommittedOffset;
+ this.logHeadOffset = logHeadOffset;
+ this.lag = logHeadOffset - consumerCommittedOffset;
+ }
- public long getConsumerCommittedOffset() {
- return consumerCommittedOffset;
- }
+ public long getConsumerCommittedOffset() {
+ return consumerCommittedOffset;
+ }
- public long getLogHeadOffset() {
- return logHeadOffset;
- }
+ public long getLogHeadOffset() {
+ return logHeadOffset;
+ }
- public long getLag() {
- return lag;
- }
+ public long getLag() {
+ return lag;
+ }
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof KafkaPartitionOffsetLag)) return false;
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof KafkaPartitionOffsetLag)) return false;
- KafkaPartitionOffsetLag that = (KafkaPartitionOffsetLag) o;
+ KafkaPartitionOffsetLag that = (KafkaPartitionOffsetLag) o;
- if (getConsumerCommittedOffset() != that.getConsumerCommittedOffset()) return false;
- if (getLogHeadOffset() != that.getLogHeadOffset()) return false;
- return getLag() == that.getLag();
+ if (getConsumerCommittedOffset() != that.getConsumerCommittedOffset()) return false;
+ if (getLogHeadOffset() != that.getLogHeadOffset()) return false;
+ return getLag() == that.getLag();
- }
+ }
- @Override
- public int hashCode() {
- int result = (int) (getConsumerCommittedOffset() ^ (getConsumerCommittedOffset() >>> 32));
- result = 31 * result + (int) (getLogHeadOffset() ^ (getLogHeadOffset() >>> 32));
- result = 31 * result + (int) (getLag() ^ (getLag() >>> 32));
- return result;
- }
+ @Override
+ public int hashCode() {
+ int result = (int) (getConsumerCommittedOffset() ^ (getConsumerCommittedOffset() >>> 32));
+ result = 31 * result + (int) (getLogHeadOffset() ^ (getLogHeadOffset() >>> 32));
+ result = 31 * result + (int) (getLag() ^ (getLag() >>> 32));
+ return result;
+ }
- @Override
- public String toString() {
- // JSONAware not working for nested element on Map so write JSON format from here
- return "{\"consumerCommittedOffset\": " + consumerCommittedOffset + ", " +
- "\"logHeadOffset\": " + logHeadOffset + ", " +
- "\"lag\": " + lag + "}";
- }
+ @Override
+ public String toString() {
+ // JSONAware not working for nested element on Map so write JSON format from here
+ return "{\"consumerCommittedOffset\": " + consumerCommittedOffset + ", " +
+ "\"logHeadOffset\": " + logHeadOffset + ", " +
+ "\"lag\": " + lag + "}";
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/224633d3/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/NewKafkaSpoutOffsetQuery.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/NewKafkaSpoutOffsetQuery.java b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/NewKafkaSpoutOffsetQuery.java
index 18bc0df..e6c4524 100644
--- a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/NewKafkaSpoutOffsetQuery.java
+++ b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/NewKafkaSpoutOffsetQuery.java
@@ -15,10 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.kafka.monitor;
/**
- * Class representing information for querying kafka for log head offsets, consumer offsets and the difference for new kafka spout using new consumer api
+ * Class representing information for querying kafka for log head offsets, consumer offsets and the difference for new kafka spout using new
+ * consumer api
*/
public class NewKafkaSpoutOffsetQuery {
private final String topics; // comma separated list of topics
@@ -52,10 +54,10 @@ public class NewKafkaSpoutOffsetQuery {
@Override
public String toString() {
return "NewKafkaSpoutOffsetQuery{" +
- "topics='" + topics + '\'' +
- ", consumerGroupId='" + consumerGroupId + '\'' +
- ", bootStrapBrokers='" + bootStrapBrokers + '\'' +
- '}';
+ "topics='" + topics + '\'' +
+ ", consumerGroupId='" + consumerGroupId + '\'' +
+ ", bootStrapBrokers='" + bootStrapBrokers + '\'' +
+ '}';
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/224633d3/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/OldKafkaSpoutOffsetQuery.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/OldKafkaSpoutOffsetQuery.java b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/OldKafkaSpoutOffsetQuery.java
index cdeed32..ea80b64 100644
--- a/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/OldKafkaSpoutOffsetQuery.java
+++ b/external/storm-kafka-monitor/src/main/java/org/apache/storm/kafka/monitor/OldKafkaSpoutOffsetQuery.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.kafka.monitor;
/**
@@ -27,7 +28,8 @@ public class OldKafkaSpoutOffsetQuery {
private final boolean isWildCardTopic; //if the topic is a wildcard
private final String brokersZkPath; //zk node prefix where kafka stores all broker information
private final String partitions; //comma separated list of partitions corresponding to leaders below (for StaticHosts)
- private final String leaders; //comma separated list of leader brokers and port corresponding to the partitions above (for StaticHosts) e.g.
+ private final String leaders;
+ //comma separated list of leader brokers and port corresponding to the partitions above (for StaticHosts) e.g.
// hostname1:9092,hostname2:9092
public OldKafkaSpoutOffsetQuery(String topic, String zkServers, String zkPath, boolean isWildCardTopic, String brokersZkPath) {
@@ -39,8 +41,9 @@ public class OldKafkaSpoutOffsetQuery {
}
- private OldKafkaSpoutOffsetQuery(String topic, String zkServers, String zkPath, boolean isWildCardTopic, String brokersZkPath, String partitions, String
- leaders) {
+ private OldKafkaSpoutOffsetQuery(String topic, String zkServers, String zkPath, boolean isWildCardTopic, String brokersZkPath,
+ String partitions, String
+ leaders) {
this.topic = topic;
this.zkServers = zkServers;
this.zkPath = zkPath;
@@ -53,14 +56,14 @@ public class OldKafkaSpoutOffsetQuery {
@Override
public String toString() {
return "OldKafkaSpoutOffsetQuery{" +
- "topic='" + topic + '\'' +
- ", zkServers='" + zkServers + '\'' +
- ", zkPath='" + zkPath + '\'' +
- ", isWildCardTopic=" + isWildCardTopic +
- ", brokersZkPath='" + brokersZkPath + '\'' +
- ", partitions='" + partitions + '\'' +
- ", leaders='" + leaders + '\'' +
- '}';
+ "topic='" + topic + '\'' +
+ ", zkServers='" + zkServers + '\'' +
+ ", zkPath='" + zkPath + '\'' +
+ ", isWildCardTopic=" + isWildCardTopic +
+ ", brokersZkPath='" + brokersZkPath + '\'' +
+ ", partitions='" + partitions + '\'' +
+ ", leaders='" + leaders + '\'' +
+ '}';
}
@Override
[48/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-kafka
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-kafka
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4fe4f04b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4fe4f04b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4fe4f04b
Branch: refs/heads/master
Commit: 4fe4f04bb9750301b96f5c20142acb9a9a6a6000
Parents: 1a2d131
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 22:59:46 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 00:22:37 2018 -0400
----------------------------------------------------------------------
external/storm-kafka/pom.xml | 2 +-
.../src/jvm/org/apache/storm/kafka/Broker.java | 51 ++--
.../jvm/org/apache/storm/kafka/BrokerHosts.java | 19 +-
.../storm/kafka/ByteBufferSerializer.java | 26 +-
.../storm/kafka/DynamicBrokersReader.java | 112 ++++----
.../kafka/DynamicPartitionConnections.java | 55 ++--
.../ExponentialBackoffMsgRetryManager.java | 75 +++---
.../storm/kafka/FailedFetchException.java | 19 +-
.../storm/kafka/FailedMsgRetryManager.java | 19 +-
.../org/apache/storm/kafka/IntSerializer.java | 24 +-
.../jvm/org/apache/storm/kafka/KafkaConfig.java | 25 +-
.../jvm/org/apache/storm/kafka/KafkaError.java | 19 +-
.../jvm/org/apache/storm/kafka/KafkaSpout.java | 58 ++--
.../jvm/org/apache/storm/kafka/KafkaUtils.java | 269 +++++++++----------
.../org/apache/storm/kafka/KeyValueScheme.java | 22 +-
.../kafka/KeyValueSchemeAsMultiScheme.java | 31 +--
.../storm/kafka/MessageMetadataScheme.java | 22 +-
.../MessageMetadataSchemeAsMultiScheme.java | 20 +-
.../jvm/org/apache/storm/kafka/Partition.java | 39 ++-
.../storm/kafka/PartitionCoordinator.java | 19 +-
.../apache/storm/kafka/PartitionManager.java | 121 +++++----
.../jvm/org/apache/storm/kafka/SpoutConfig.java | 19 +-
.../apache/storm/kafka/StaticCoordinator.java | 27 +-
.../jvm/org/apache/storm/kafka/StaticHosts.java | 19 +-
.../storm/kafka/StaticPartitionConnections.java | 22 +-
.../storm/kafka/StringKeyValueScheme.java | 24 +-
.../kafka/StringMessageAndMetadataScheme.java | 27 +-
.../storm/kafka/StringMultiSchemeWithTopic.java | 28 +-
.../org/apache/storm/kafka/StringScheme.java | 38 ++-
.../kafka/TopicOffsetOutOfRangeException.java | 1 +
.../org/apache/storm/kafka/ZkCoordinator.java | 50 ++--
.../src/jvm/org/apache/storm/kafka/ZkHosts.java | 19 +-
.../src/jvm/org/apache/storm/kafka/ZkState.java | 65 +++--
.../org/apache/storm/kafka/bolt/KafkaBolt.java | 69 +++--
.../FieldNameBasedTupleToKafkaMapper.java | 21 +-
.../kafka/bolt/mapper/TupleToKafkaMapper.java | 25 +-
.../bolt/selector/DefaultTopicSelector.java | 19 +-
.../bolt/selector/FieldIndexTopicSelector.java | 19 +-
.../bolt/selector/FieldNameTopicSelector.java | 19 +-
.../kafka/bolt/selector/KafkaTopicSelector.java | 22 +-
.../apache/storm/kafka/trident/Coordinator.java | 27 +-
.../storm/kafka/trident/DefaultCoordinator.java | 19 +-
.../trident/GlobalPartitionInformation.java | 41 ++-
.../storm/kafka/trident/IBatchCoordinator.java | 19 +-
.../storm/kafka/trident/IBrokerReader.java | 20 +-
.../apache/storm/kafka/trident/MaxMetric.java | 19 +-
.../kafka/trident/OpaqueTridentKafkaSpout.java | 44 ++-
.../storm/kafka/trident/StaticBrokerReader.java | 23 +-
.../trident/TransactionalTridentKafkaSpout.java | 22 +-
.../storm/kafka/trident/TridentKafkaConfig.java | 19 +-
.../kafka/trident/TridentKafkaEmitter.java | 73 +++--
.../storm/kafka/trident/TridentKafkaState.java | 49 ++--
.../kafka/trident/TridentKafkaStateFactory.java | 34 +--
.../kafka/trident/TridentKafkaUpdater.java | 22 +-
.../storm/kafka/trident/ZkBrokerReader.java | 113 ++++----
.../FieldNameBasedTupleToKafkaMapper.java | 19 +-
.../mapper/TridentTupleToKafkaMapper.java | 26 +-
.../trident/selector/DefaultTopicSelector.java | 19 +-
.../trident/selector/KafkaTopicSelector.java | 22 +-
.../storm/kafka/DynamicBrokersReaderTest.java | 39 ++-
.../ExponentialBackoffMsgRetryManagerTest.java | 49 ++--
.../org/apache/storm/kafka/KafkaErrorTest.java | 19 +-
.../org/apache/storm/kafka/KafkaTestBroker.java | 29 +-
.../org/apache/storm/kafka/KafkaUtilsTest.java | 73 +++--
.../storm/kafka/PartitionManagerTest.java | 34 +--
.../storm/kafka/StringKeyValueSchemeTest.java | 36 ++-
.../apache/storm/kafka/TestStringScheme.java | 28 +-
.../test/org/apache/storm/kafka/TestUtils.java | 32 ++-
.../apache/storm/kafka/TridentKafkaTest.java | 40 ++-
.../apache/storm/kafka/ZkCoordinatorTest.java | 49 ++--
.../apache/storm/kafka/bolt/KafkaBoltTest.java | 159 ++++++-----
71 files changed, 1246 insertions(+), 1551 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-kafka/pom.xml b/external/storm-kafka/pom.xml
index d3b38db..2e0f35b 100644
--- a/external/storm-kafka/pom.xml
+++ b/external/storm-kafka/pom.xml
@@ -57,7 +57,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>557</maxAllowedViolations>
+ <maxAllowedViolations>180</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/Broker.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/Broker.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/Broker.java
index 0d95e8d..b33af99 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/Broker.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/Broker.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import com.google.common.base.Objects;
-
import java.io.Serializable;
public class Broker implements Serializable, Comparable<Broker> {
@@ -27,9 +21,9 @@ public class Broker implements Serializable, Comparable<Broker> {
// for kryo compatibility
private Broker() {
-
+
}
-
+
public Broker(String host, int port) {
this.host = host;
this.port = port;
@@ -39,6 +33,19 @@ public class Broker implements Serializable, Comparable<Broker> {
this(host, 9092);
}
+ public static Broker fromString(String host) {
+ Broker hp;
+ String[] spec = host.split(":");
+ if (spec.length == 1) {
+ hp = new Broker(spec[0]);
+ } else if (spec.length == 2) {
+ hp = new Broker(spec[0], Integer.parseInt(spec[1]));
+ } else {
+ throw new IllegalArgumentException("Invalid host specification: " + host);
+ }
+ return hp;
+ }
+
@Override
public int hashCode() {
return Objects.hashCode(host, port);
@@ -61,20 +68,6 @@ public class Broker implements Serializable, Comparable<Broker> {
return host + ":" + port;
}
- public static Broker fromString(String host) {
- Broker hp;
- String[] spec = host.split(":");
- if (spec.length == 1) {
- hp = new Broker(spec[0]);
- } else if (spec.length == 2) {
- hp = new Broker(spec[0], Integer.parseInt(spec[1]));
- } else {
- throw new IllegalArgumentException("Invalid host specification: " + host);
- }
- return hp;
- }
-
-
@Override
public int compareTo(Broker o) {
if (this.host.equals(o.host)) {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/BrokerHosts.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/BrokerHosts.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/BrokerHosts.java
index 13ba0a1..dbd6a10 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/BrokerHosts.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/BrokerHosts.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ByteBufferSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/ByteBufferSerializer.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ByteBufferSerializer.java
index 2a18a7f..37986a5 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/ByteBufferSerializer.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ByteBufferSerializer.java
@@ -15,27 +15,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.kafka;
-import org.apache.storm.utils.Utils;
-import org.apache.kafka.common.serialization.Serializer;
+package org.apache.storm.kafka;
import java.nio.ByteBuffer;
import java.util.Map;
+import org.apache.kafka.common.serialization.Serializer;
+import org.apache.storm.utils.Utils;
public class ByteBufferSerializer implements Serializer<ByteBuffer> {
- @Override
- public void configure(Map<String, ?> map, boolean b) {
+ @Override
+ public void configure(Map<String, ?> map, boolean b) {
- }
+ }
- @Override
- public void close() {
+ @Override
+ public void close() {
- }
+ }
- @Override
- public byte[] serialize(String s, ByteBuffer b) {
- return Utils.toByteArray(b);
- }
+ @Override
+ public byte[] serialize(String s, ByteBuffer b) {
+ return Utils.toByteArray(b);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/DynamicBrokersReader.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/DynamicBrokersReader.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/DynamicBrokersReader.java
index c203359..49ad530 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/DynamicBrokersReader.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/DynamicBrokersReader.java
@@ -1,37 +1,31 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
-import org.apache.storm.Config;
-import org.apache.storm.utils.ObjectReader;
import com.google.common.base.Preconditions;
+import java.net.SocketTimeoutException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryNTimes;
+import org.apache.storm.Config;
+import org.apache.storm.kafka.trident.GlobalPartitionInformation;
+import org.apache.storm.utils.ObjectReader;
import org.json.simple.JSONValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.storm.kafka.trident.GlobalPartitionInformation;
-
-import java.net.SocketTimeoutException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
public class DynamicBrokersReader {
@@ -49,7 +43,7 @@ public class DynamicBrokersReader {
validateConfig(conf);
- Preconditions.checkNotNull(zkStr,"zkString cannot be null");
+ Preconditions.checkNotNull(zkStr, "zkString cannot be null");
Preconditions.checkNotNull(zkPath, "zkPath cannot be null");
Preconditions.checkNotNull(topic, "topic cannot be null");
@@ -58,11 +52,11 @@ public class DynamicBrokersReader {
_isWildcardTopic = ObjectReader.getBoolean(conf.get("kafka.topic.wildcard.match"), false);
try {
_curator = CuratorFrameworkFactory.newClient(
- zkStr,
- ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)),
- ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT)),
- new RetryNTimes(ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)),
- ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL))));
+ zkStr,
+ ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)),
+ ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT)),
+ new RetryNTimes(ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)),
+ ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL))));
_curator.start();
} catch (Exception ex) {
LOG.error("Couldn't connect to zookeeper", ex);
@@ -74,33 +68,33 @@ public class DynamicBrokersReader {
* Get all partitions with their current leaders
*/
public List<GlobalPartitionInformation> getBrokerInfo() throws SocketTimeoutException {
- List<String> topics = getTopics();
- List<GlobalPartitionInformation> partitions = new ArrayList<GlobalPartitionInformation>();
-
- for (String topic : topics) {
- GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(topic, this._isWildcardTopic);
- try {
- int numPartitionsForTopic = getNumPartitions(topic);
- String brokerInfoPath = brokerPath();
- for (int partition = 0; partition < numPartitionsForTopic; partition++) {
- int leader = getLeaderFor(topic,partition);
- String path = brokerInfoPath + "/" + leader;
- try {
- byte[] brokerData = _curator.getData().forPath(path);
- Broker hp = getBrokerHost(brokerData);
- globalPartitionInformation.addPartition(partition, hp);
- } catch (org.apache.zookeeper.KeeperException.NoNodeException e) {
- LOG.error("Node {} does not exist ", path);
- }
- }
- } catch (SocketTimeoutException e) {
- throw e;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- LOG.info("Read partition info from zookeeper: " + globalPartitionInformation);
- partitions.add(globalPartitionInformation);
- }
+ List<String> topics = getTopics();
+ List<GlobalPartitionInformation> partitions = new ArrayList<GlobalPartitionInformation>();
+
+ for (String topic : topics) {
+ GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(topic, this._isWildcardTopic);
+ try {
+ int numPartitionsForTopic = getNumPartitions(topic);
+ String brokerInfoPath = brokerPath();
+ for (int partition = 0; partition < numPartitionsForTopic; partition++) {
+ int leader = getLeaderFor(topic, partition);
+ String path = brokerInfoPath + "/" + leader;
+ try {
+ byte[] brokerData = _curator.getData().forPath(path);
+ Broker hp = getBrokerHost(brokerData);
+ globalPartitionInformation.addPartition(partition, hp);
+ } catch (org.apache.zookeeper.KeeperException.NoNodeException e) {
+ LOG.error("Node {} does not exist ", path);
+ }
+ }
+ } catch (SocketTimeoutException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ LOG.info("Read partition info from zookeeper: " + globalPartitionInformation);
+ partitions.add(globalPartitionInformation);
+ }
return partitions;
}
@@ -135,9 +129,10 @@ public class DynamicBrokersReader {
}
}
- public String topicsPath () {
+ public String topicsPath() {
return _zkPath + "/topics";
}
+
public String partitionPath(String topic) {
return topicsPath() + "/" + topic + "/partitions";
}
@@ -147,7 +142,6 @@ public class DynamicBrokersReader {
}
-
/**
* get /brokers/topics/distributedTopic/partitions/1/state
* { "controller_epoch":4, "isr":[ 1, 0 ], "leader":1, "leader_epoch":1, "version":1 }
@@ -202,13 +196,13 @@ public class DynamicBrokersReader {
*/
private void validateConfig(final Map<String, Object> conf) {
Preconditions.checkNotNull(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT),
- "%s cannot be null", Config.STORM_ZOOKEEPER_SESSION_TIMEOUT);
+ "%s cannot be null", Config.STORM_ZOOKEEPER_SESSION_TIMEOUT);
Preconditions.checkNotNull(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT),
- "%s cannot be null", Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT);
+ "%s cannot be null", Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT);
Preconditions.checkNotNull(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES),
- "%s cannot be null", Config.STORM_ZOOKEEPER_RETRY_TIMES);
+ "%s cannot be null", Config.STORM_ZOOKEEPER_RETRY_TIMES);
Preconditions.checkNotNull(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL),
- "%s cannot be null", Config.STORM_ZOOKEEPER_RETRY_INTERVAL);
+ "%s cannot be null", Config.STORM_ZOOKEEPER_RETRY_INTERVAL);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/DynamicPartitionConnections.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/DynamicPartitionConnections.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/DynamicPartitionConnections.java
index 4c5dba5..1ca7144 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/DynamicPartitionConnections.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/DynamicPartitionConnections.java
@@ -1,50 +1,33 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka;
-import kafka.javaapi.consumer.SimpleConsumer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.storm.kafka.trident.IBrokerReader;
+package org.apache.storm.kafka;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import kafka.javaapi.consumer.SimpleConsumer;
+import org.apache.storm.kafka.trident.IBrokerReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class DynamicPartitionConnections {
private static final Logger LOG = LoggerFactory.getLogger(DynamicPartitionConnections.class);
-
- static class ConnectionInfo {
- SimpleConsumer consumer;
- Set<String> partitions = new HashSet<String>();
-
- public ConnectionInfo(SimpleConsumer consumer) {
- this.consumer = consumer;
- }
- }
-
Map<Broker, ConnectionInfo> _connections = new HashMap<>();
KafkaConfig _config;
IBrokerReader _reader;
-
public DynamicPartitionConnections(KafkaConfig config, IBrokerReader brokerReader) {
_config = config;
_reader = brokerReader;
@@ -57,10 +40,11 @@ public class DynamicPartitionConnections {
public SimpleConsumer register(Broker host, String topic, int partition) {
if (!_connections.containsKey(host)) {
- _connections.put(host, new ConnectionInfo(new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId)));
+ _connections.put(host, new ConnectionInfo(
+ new SimpleConsumer(host.host, host.port, _config.socketTimeoutMs, _config.bufferSizeBytes, _config.clientId)));
}
ConnectionInfo info = _connections.get(host);
- info.partitions.add(getHashKey(topic,partition));
+ info.partitions.add(getHashKey(topic, partition));
return info.consumer;
}
@@ -74,7 +58,7 @@ public class DynamicPartitionConnections {
public void unregister(Broker port, String topic, int partition) {
ConnectionInfo info = _connections.get(port);
- info.partitions.remove(getHashKey(topic,partition));
+ info.partitions.remove(getHashKey(topic, partition));
if (info.partitions.isEmpty()) {
info.consumer.close();
_connections.remove(port);
@@ -95,4 +79,13 @@ public class DynamicPartitionConnections {
private String getHashKey(String topic, int partition) {
return topic + "_" + partition;
}
+
+ static class ConnectionInfo {
+ SimpleConsumer consumer;
+ Set<String> partitions = new HashSet<String>();
+
+ public ConnectionInfo(SimpleConsumer consumer) {
+ this.consumer = consumer;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ExponentialBackoffMsgRetryManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/ExponentialBackoffMsgRetryManager.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ExponentialBackoffMsgRetryManager.java
index 60654a5..2651c30 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/ExponentialBackoffMsgRetryManager.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/ExponentialBackoffMsgRetryManager.java
@@ -1,23 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
-import org.apache.storm.utils.Time;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Map;
@@ -25,6 +19,7 @@ import java.util.PriorityQueue;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.storm.utils.Time;
public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager {
@@ -34,7 +29,7 @@ public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager
private int retryLimit;
private Queue<MessageRetryRecord> waiting;
- private Map<Long,MessageRetryRecord> records;
+ private Map<Long, MessageRetryRecord> records;
public ExponentialBackoffMsgRetryManager() {
@@ -46,15 +41,15 @@ public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager
this.retryDelayMaxMs = spoutConfig.retryDelayMaxMs;
this.retryLimit = spoutConfig.retryLimit;
this.waiting = new PriorityQueue<MessageRetryRecord>(11, new RetryTimeComparator());
- this.records = new ConcurrentHashMap<Long,MessageRetryRecord>();
+ this.records = new ConcurrentHashMap<Long, MessageRetryRecord>();
}
@Override
public void failed(Long offset) {
MessageRetryRecord oldRecord = this.records.get(offset);
MessageRetryRecord newRecord = oldRecord == null ?
- new MessageRetryRecord(offset) :
- oldRecord.createNextRetryRecord();
+ new MessageRetryRecord(offset) :
+ oldRecord.createNextRetryRecord();
this.records.put(offset, newRecord);
this.waiting.add(newRecord);
}
@@ -98,16 +93,16 @@ public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager
public boolean shouldReEmitMsg(Long offset) {
MessageRetryRecord record = this.records.get(offset);
return record != null &&
- this.waiting.contains(record) &&
- Time.currentTimeMillis() >= record.retryTimeUTC;
+ this.waiting.contains(record) &&
+ Time.currentTimeMillis() >= record.retryTimeUTC;
}
@Override
public boolean retryFurther(Long offset) {
MessageRetryRecord record = this.records.get(offset);
- return ! (record != null &&
- this.retryLimit > 0 &&
- this.retryLimit <= record.retryNum);
+ return !(record != null &&
+ this.retryLimit > 0 &&
+ this.retryLimit <= record.retryNum);
}
@Override
@@ -117,9 +112,9 @@ public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager
@Override
public Set<Long> clearOffsetsBefore(Long kafkaOffset) {
- Set<Long> invalidOffsets = new HashSet<Long>();
- for(Long offset : records.keySet()){
- if(offset < kafkaOffset){
+ Set<Long> invalidOffsets = new HashSet<Long>();
+ for (Long offset : records.keySet()) {
+ if (offset < kafkaOffset) {
MessageRetryRecord record = this.records.remove(offset);
if (record != null) {
this.waiting.remove(record);
@@ -130,6 +125,19 @@ public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager
return invalidOffsets;
}
+ private static class RetryTimeComparator implements Comparator<MessageRetryRecord> {
+
+ @Override
+ public int compare(MessageRetryRecord record1, MessageRetryRecord record2) {
+ return Long.valueOf(record1.retryTimeUTC).compareTo(Long.valueOf(record2.retryTimeUTC));
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return false;
+ }
+ }
+
/**
* A MessageRetryRecord holds the data of how many times a message has
* failed and been retried, and when the last failure occurred. It can
@@ -174,8 +182,8 @@ public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager
double delay = retryInitialDelayMs * delayMultiplier;
Long maxLong = Long.MAX_VALUE;
long delayThisRetryMs = delay >= maxLong.doubleValue()
- ? maxLong
- : (long) delay;
+ ? maxLong
+ : (long) delay;
return Math.min(delayThisRetryMs, retryDelayMaxMs);
}
@@ -190,17 +198,4 @@ public class ExponentialBackoffMsgRetryManager implements FailedMsgRetryManager
return Long.valueOf(this.offset).hashCode();
}
}
-
- private static class RetryTimeComparator implements Comparator<MessageRetryRecord> {
-
- @Override
- public int compare(MessageRetryRecord record1, MessageRetryRecord record2) {
- return Long.valueOf(record1.retryTimeUTC).compareTo(Long.valueOf(record2.retryTimeUTC));
- }
-
- @Override
- public boolean equals(Object obj) {
- return false;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/FailedFetchException.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/FailedFetchException.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/FailedFetchException.java
index 448d0c3..a1f3fe5 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/FailedFetchException.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/FailedFetchException.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
public class FailedFetchException extends RuntimeException {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/FailedMsgRetryManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/FailedMsgRetryManager.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/FailedMsgRetryManager.java
index c7a7a04..b7fafdc 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/FailedMsgRetryManager.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/FailedMsgRetryManager.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/IntSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/IntSerializer.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/IntSerializer.java
index 7cdfc87..fef8625 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/IntSerializer.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/IntSerializer.java
@@ -15,28 +15,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.kafka;
-import org.apache.kafka.common.serialization.Serializer;
+package org.apache.storm.kafka;
import java.util.Map;
+import org.apache.kafka.common.serialization.Serializer;
public class IntSerializer implements Serializer<Integer> {
- @Override
- public void configure(Map<String, ?> map, boolean b) {
- }
+ @Override
+ public void configure(Map<String, ?> map, boolean b) {
+ }
- @Override
- public byte[] serialize(String topic, Integer val) {
- return new byte[] {
+ @Override
+ public byte[] serialize(String topic, Integer val) {
+ return new byte[]{
(byte) (val >>> 24),
(byte) (val >>> 16),
(byte) (val >>> 8),
val.byteValue()
};
- }
+ }
- @Override
- public void close() {
- }
+ @Override
+ public void close() {
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaConfig.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaConfig.java
index 1c9ada8..a93f426 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaConfig.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaConfig.java
@@ -1,32 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import java.io.Serializable;
-
+import kafka.api.FetchRequest;
import org.apache.storm.spout.MultiScheme;
import org.apache.storm.spout.RawMultiScheme;
-import kafka.api.FetchRequest;
-
public class KafkaConfig implements Serializable {
private static final long serialVersionUID = 5276718734571623855L;
-
+
public final BrokerHosts hosts;
public final String topic;
public final String clientId;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaError.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaError.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaError.java
index 1d866e7..4bf2ed2 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaError.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaError.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
public enum KafkaError {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaSpout.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaSpout.java
index 7b1243c..aed5986 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaSpout.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaSpout.java
@@ -1,24 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import com.google.common.base.Strings;
-
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.storm.Config;
import org.apache.storm.kafka.PartitionManager.KafkaMessageId;
import org.apache.storm.kafka.trident.GlobalPartitionInformation;
@@ -30,27 +29,16 @@ import org.apache.storm.topology.base.BaseRichSpout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
-
// TODO: need to add blacklisting
// TODO: need to make a best effort to not re-emit messages if don't have to
public class KafkaSpout extends BaseRichSpout {
- static enum EmitState {
- EMITTED_MORE_LEFT,
- EMITTED_END,
- NO_EMITTED
- }
-
private static final Logger LOG = LoggerFactory.getLogger(KafkaSpout.class);
-
SpoutConfig _spoutConfig;
SpoutOutputCollector _collector;
PartitionCoordinator _coordinator;
DynamicPartitionConnections _connections;
ZkState _state;
-
long _lastUpdateMs = 0;
-
int _currPartitionIndex = 0;
public KafkaSpout(SpoutConfig spoutConf) {
@@ -81,12 +69,12 @@ public class KafkaSpout extends BaseRichSpout {
int totalTasks = context.getComponentTasks(context.getThisComponentId()).size();
if (_spoutConfig.hosts instanceof StaticHosts) {
_coordinator = new StaticCoordinator(_connections, conf,
- _spoutConfig, _state, context.getThisTaskIndex(),
- totalTasks, context.getThisTaskId(), topologyInstanceId);
+ _spoutConfig, _state, context.getThisTaskIndex(),
+ totalTasks, context.getThisTaskId(), topologyInstanceId);
} else {
_coordinator = new ZkCoordinator(_connections, conf,
- _spoutConfig, _state, context.getThisTaskIndex(),
- totalTasks, context.getThisTaskId(), topologyInstanceId);
+ _spoutConfig, _state, context.getThisTaskIndex(),
+ totalTasks, context.getThisTaskId(), topologyInstanceId);
}
context.registerMetric("kafkaOffset", new IMetric() {
@@ -158,7 +146,7 @@ public class KafkaSpout extends BaseRichSpout {
}
private PartitionManager getManagerForPartition(int partition) {
- for (PartitionManager partitionManager: _coordinator.getMyManagedPartitions()) {
+ for (PartitionManager partitionManager : _coordinator.getMyManagedPartitions()) {
if (partitionManager.getPartition().partition == partition) {
return partitionManager;
}
@@ -203,7 +191,7 @@ public class KafkaSpout extends BaseRichSpout {
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
- if (!Strings.isNullOrEmpty(_spoutConfig.outputStreamId)) {
+ if (!Strings.isNullOrEmpty(_spoutConfig.outputStreamId)) {
declarer.declareStream(_spoutConfig.outputStreamId, _spoutConfig.scheme.getOutputFields());
} else {
declarer.declare(_spoutConfig.scheme.getOutputFields());
@@ -211,7 +199,7 @@ public class KafkaSpout extends BaseRichSpout {
}
@Override
- public Map<String, Object> getComponentConfiguration () {
+ public Map<String, Object> getComponentConfiguration() {
Map<String, Object> configuration = super.getComponentConfiguration();
if (configuration == null) {
configuration = new HashMap<>();
@@ -240,7 +228,7 @@ public class KafkaSpout extends BaseRichSpout {
List<Partition> partitions = globalPartitionInformation.getOrderedPartitions();
StringBuilder staticPartitions = new StringBuilder();
StringBuilder leaderHosts = new StringBuilder();
- for (Partition partition: partitions) {
+ for (Partition partition : partitions) {
staticPartitions.append(partition.partition + ",");
leaderHosts.append(partition.host.host + ":" + partition.host.port).append(",");
}
@@ -258,4 +246,10 @@ public class KafkaSpout extends BaseRichSpout {
}
}
+ static enum EmitState {
+ EMITTED_MORE_LEFT,
+ EMITTED_END,
+ NO_EMITTED
+ }
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaUtils.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaUtils.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaUtils.java
index 76bb896..38958b2 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaUtils.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KafkaUtils.java
@@ -1,32 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import com.google.common.base.Preconditions;
-
-import org.apache.storm.kafka.trident.GlobalPartitionInformation;
-import org.apache.storm.kafka.trident.IBrokerReader;
-import org.apache.storm.kafka.trident.StaticBrokerReader;
-import org.apache.storm.kafka.trident.ZkBrokerReader;
-import org.apache.storm.metric.api.IMetric;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
@@ -39,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-
import kafka.api.FetchRequest;
import kafka.api.FetchRequestBuilder;
import kafka.api.PartitionOffsetRequestInfo;
@@ -49,6 +34,13 @@ import kafka.javaapi.OffsetRequest;
import kafka.javaapi.consumer.SimpleConsumer;
import kafka.javaapi.message.ByteBufferMessageSet;
import kafka.message.Message;
+import org.apache.storm.kafka.trident.GlobalPartitionInformation;
+import org.apache.storm.kafka.trident.IBrokerReader;
+import org.apache.storm.kafka.trident.StaticBrokerReader;
+import org.apache.storm.kafka.trident.ZkBrokerReader;
+import org.apache.storm.metric.api.IMetric;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class KafkaUtils {
@@ -57,7 +49,7 @@ public class KafkaUtils {
private static final int NO_OFFSET = -5;
//suppress default constructor for noninstantiablility
- private KafkaUtils(){
+ private KafkaUtils() {
throw new AssertionError();
}
@@ -80,7 +72,7 @@ public class KafkaUtils {
Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
OffsetRequest request = new OffsetRequest(
- requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
+ requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition);
if (offsets.length > 0) {
@@ -90,121 +82,23 @@ public class KafkaUtils {
}
}
- public static class KafkaOffsetMetric implements IMetric {
- Map<Partition, PartitionManager.OffsetData> _partitionToOffset = new HashMap<Partition, PartitionManager.OffsetData>();
- Set<Partition> _partitions;
- DynamicPartitionConnections _connections;
-
- public KafkaOffsetMetric(DynamicPartitionConnections connections) {
- _connections = connections;
- }
-
- public void setOffsetData(Partition partition, PartitionManager.OffsetData offsetData) {
- _partitionToOffset.put(partition, offsetData);
- }
-
- private class TopicMetrics {
- long totalSpoutLag = 0;
- long totalEarliestTimeOffset = 0;
- long totalLatestTimeOffset = 0;
- long totalLatestEmittedOffset = 0;
- long totalLatestCompletedOffset = 0;
- }
-
- @Override
- public Object getValueAndReset() {
- try {
- HashMap<String, Long> ret = new HashMap<>();
- if (_partitions != null && _partitions.size() == _partitionToOffset.size()) {
- Map<String,TopicMetrics> topicMetricsMap = new TreeMap<String, TopicMetrics>();
- for (Map.Entry<Partition, PartitionManager.OffsetData> e : _partitionToOffset.entrySet()) {
- Partition partition = e.getKey();
- SimpleConsumer consumer = _connections.getConnection(partition);
- if (consumer == null) {
- LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?");
- return null;
- }
- long latestTimeOffset = getOffset(consumer, partition.topic, partition.partition, kafka.api.OffsetRequest.LatestTime());
- long earliestTimeOffset = getOffset(consumer, partition.topic, partition.partition, kafka.api.OffsetRequest.EarliestTime());
- if (latestTimeOffset == KafkaUtils.NO_OFFSET) {
- LOG.warn("No data found in Kafka Partition " + partition.getId());
- return null;
- }
- long latestEmittedOffset = e.getValue().latestEmittedOffset;
- long latestCompletedOffset = e.getValue().latestCompletedOffset;
- long spoutLag = latestTimeOffset - latestCompletedOffset;
- String topic = partition.topic;
- String metricPath = partition.getId();
- //Handle the case where Partition Path Id does not contain topic name Partition.getId() == "partition_" + partition
- if (!metricPath.startsWith(topic + "/")) {
- metricPath = topic + "/" + metricPath;
- }
- ret.put(metricPath + "/" + "spoutLag", spoutLag);
- ret.put(metricPath + "/" + "earliestTimeOffset", earliestTimeOffset);
- ret.put(metricPath + "/" + "latestTimeOffset", latestTimeOffset);
- ret.put(metricPath + "/" + "latestEmittedOffset", latestEmittedOffset);
- ret.put(metricPath + "/" + "latestCompletedOffset", latestCompletedOffset);
-
- if (!topicMetricsMap.containsKey(partition.topic)) {
- topicMetricsMap.put(partition.topic,new TopicMetrics());
- }
-
- TopicMetrics topicMetrics = topicMetricsMap.get(partition.topic);
- topicMetrics.totalSpoutLag += spoutLag;
- topicMetrics.totalEarliestTimeOffset += earliestTimeOffset;
- topicMetrics.totalLatestTimeOffset += latestTimeOffset;
- topicMetrics.totalLatestEmittedOffset += latestEmittedOffset;
- topicMetrics.totalLatestCompletedOffset += latestCompletedOffset;
- }
-
- for(Map.Entry<String, TopicMetrics> e : topicMetricsMap.entrySet()) {
- String topic = e.getKey();
- TopicMetrics topicMetrics = e.getValue();
- ret.put(topic + "/" + "totalSpoutLag", topicMetrics.totalSpoutLag);
- ret.put(topic + "/" + "totalEarliestTimeOffset", topicMetrics.totalEarliestTimeOffset);
- ret.put(topic + "/" + "totalLatestTimeOffset", topicMetrics.totalLatestTimeOffset);
- ret.put(topic + "/" + "totalLatestEmittedOffset", topicMetrics.totalLatestEmittedOffset);
- ret.put(topic + "/" + "totalLatestCompletedOffset", topicMetrics.totalLatestCompletedOffset);
- }
-
- return ret;
- } else {
- LOG.info("Metrics Tick: Not enough data to calculate spout lag.");
- }
- } catch (Throwable t) {
- LOG.warn("Metrics Tick: Exception when computing kafkaOffset metric.", t);
- }
- return null;
- }
-
- public void refreshPartitions(Set<Partition> partitions) {
- _partitions = partitions;
- Iterator<Partition> it = _partitionToOffset.keySet().iterator();
- while (it.hasNext()) {
- if (!partitions.contains(it.next())) {
- it.remove();
- }
- }
- }
- }
-
public static ByteBufferMessageSet fetchMessages(KafkaConfig config, SimpleConsumer consumer, Partition partition, long offset)
- throws TopicOffsetOutOfRangeException, FailedFetchException,RuntimeException {
+ throws TopicOffsetOutOfRangeException, FailedFetchException, RuntimeException {
ByteBufferMessageSet msgs = null;
String topic = partition.topic;
int partitionId = partition.partition;
FetchRequestBuilder builder = new FetchRequestBuilder();
FetchRequest fetchRequest = builder.addFetch(topic, partitionId, offset, config.fetchSizeBytes).
- clientId(config.clientId).maxWait(config.fetchMaxWait).minBytes(config.minFetchByte).build();
+ clientId(config.clientId).maxWait(config.fetchMaxWait).minBytes(config.minFetchByte).build();
FetchResponse fetchResponse;
try {
fetchResponse = consumer.fetch(fetchRequest);
} catch (Exception e) {
if (e instanceof ConnectException ||
- e instanceof SocketTimeoutException ||
- e instanceof IOException ||
- e instanceof UnresolvedAddressException
- ) {
+ e instanceof SocketTimeoutException ||
+ e instanceof IOException ||
+ e instanceof UnresolvedAddressException
+ ) {
LOG.warn("Network error when fetching messages:", e);
throw new FailedFetchException(e);
} else {
@@ -225,11 +119,11 @@ public class KafkaUtils {
} else {
msgs = fetchResponse.messageSet(topic, partitionId);
}
- LOG.debug("Messages fetched. [config = {}], [consumer = {}], [partition = {}], [offset = {}], [msgs = {}]", config, consumer, partition, offset, msgs);
+ LOG.debug("Messages fetched. [config = {}], [consumer = {}], [partition = {}], [offset = {}], [msgs = {}]", config, consumer,
+ partition, offset, msgs);
return msgs;
}
-
public static Iterable<List<Object>> generateTuples(KafkaConfig kafkaConfig, Message msg, String topic) {
Iterable<List<Object>> tups;
ByteBuffer payload = msg.payload();
@@ -241,15 +135,16 @@ public class KafkaUtils {
tups = ((KeyValueSchemeAsMultiScheme) kafkaConfig.scheme).deserializeKeyAndValue(key, payload);
} else {
if (kafkaConfig.scheme instanceof StringMultiSchemeWithTopic) {
- tups = ((StringMultiSchemeWithTopic)kafkaConfig.scheme).deserializeWithTopic(topic, payload);
+ tups = ((StringMultiSchemeWithTopic) kafkaConfig.scheme).deserializeWithTopic(topic, payload);
} else {
tups = kafkaConfig.scheme.deserialize(payload);
}
}
return tups;
}
-
- public static Iterable<List<Object>> generateTuples(MessageMetadataSchemeAsMultiScheme scheme, Message msg, Partition partition, long offset) {
+
+ public static Iterable<List<Object>> generateTuples(MessageMetadataSchemeAsMultiScheme scheme, Message msg, Partition partition,
+ long offset) {
ByteBuffer payload = msg.payload();
if (payload == null) {
return null;
@@ -257,18 +152,18 @@ public class KafkaUtils {
return scheme.deserializeMessageWithMetadata(payload, partition, offset);
}
-
public static List<Partition> calculatePartitionsForTask(List<GlobalPartitionInformation> partitons,
- int totalTasks, int taskIndex, int taskId) {
+ int totalTasks, int taskIndex, int taskId) {
Preconditions.checkArgument(taskIndex < totalTasks, "task index must be less that total tasks");
List<Partition> taskPartitions = new ArrayList<Partition>();
List<Partition> partitions = new ArrayList<Partition>();
- for(GlobalPartitionInformation partitionInformation : partitons) {
+ for (GlobalPartitionInformation partitionInformation : partitons) {
partitions.addAll(partitionInformation.getOrderedPartitions());
}
int numPartitions = partitions.size();
if (numPartitions < totalTasks) {
- LOG.warn("there are more tasks than partitions (tasks: " + totalTasks + "; partitions: " + numPartitions + "), some tasks will be idle");
+ LOG.warn("there are more tasks than partitions (tasks: " + totalTasks + "; partitions: " + numPartitions +
+ "), some tasks will be idle");
}
for (int i = taskIndex; i < numPartitions; i += totalTasks) {
Partition taskPartition = partitions.get(i);
@@ -290,4 +185,104 @@ public class KafkaUtils {
public static String taskPrefix(int taskIndex, int totalTasks, int taskId) {
return "Task [" + (taskIndex + 1) + "/" + totalTasks + "], Task-ID: " + taskId;
}
+
+ public static class KafkaOffsetMetric implements IMetric {
+ Map<Partition, PartitionManager.OffsetData> _partitionToOffset = new HashMap<Partition, PartitionManager.OffsetData>();
+ Set<Partition> _partitions;
+ DynamicPartitionConnections _connections;
+
+ public KafkaOffsetMetric(DynamicPartitionConnections connections) {
+ _connections = connections;
+ }
+
+ public void setOffsetData(Partition partition, PartitionManager.OffsetData offsetData) {
+ _partitionToOffset.put(partition, offsetData);
+ }
+
+ @Override
+ public Object getValueAndReset() {
+ try {
+ HashMap<String, Long> ret = new HashMap<>();
+ if (_partitions != null && _partitions.size() == _partitionToOffset.size()) {
+ Map<String, TopicMetrics> topicMetricsMap = new TreeMap<String, TopicMetrics>();
+ for (Map.Entry<Partition, PartitionManager.OffsetData> e : _partitionToOffset.entrySet()) {
+ Partition partition = e.getKey();
+ SimpleConsumer consumer = _connections.getConnection(partition);
+ if (consumer == null) {
+ LOG.warn("partitionToOffset contains partition not found in _connections. Stale partition data?");
+ return null;
+ }
+ long latestTimeOffset =
+ getOffset(consumer, partition.topic, partition.partition, kafka.api.OffsetRequest.LatestTime());
+ long earliestTimeOffset =
+ getOffset(consumer, partition.topic, partition.partition, kafka.api.OffsetRequest.EarliestTime());
+ if (latestTimeOffset == KafkaUtils.NO_OFFSET) {
+ LOG.warn("No data found in Kafka Partition " + partition.getId());
+ return null;
+ }
+ long latestEmittedOffset = e.getValue().latestEmittedOffset;
+ long latestCompletedOffset = e.getValue().latestCompletedOffset;
+ long spoutLag = latestTimeOffset - latestCompletedOffset;
+ String topic = partition.topic;
+ String metricPath = partition.getId();
+ //Handle the case where Partition Path Id does not contain topic name Partition.getId() == "partition_" + partition
+ if (!metricPath.startsWith(topic + "/")) {
+ metricPath = topic + "/" + metricPath;
+ }
+ ret.put(metricPath + "/" + "spoutLag", spoutLag);
+ ret.put(metricPath + "/" + "earliestTimeOffset", earliestTimeOffset);
+ ret.put(metricPath + "/" + "latestTimeOffset", latestTimeOffset);
+ ret.put(metricPath + "/" + "latestEmittedOffset", latestEmittedOffset);
+ ret.put(metricPath + "/" + "latestCompletedOffset", latestCompletedOffset);
+
+ if (!topicMetricsMap.containsKey(partition.topic)) {
+ topicMetricsMap.put(partition.topic, new TopicMetrics());
+ }
+
+ TopicMetrics topicMetrics = topicMetricsMap.get(partition.topic);
+ topicMetrics.totalSpoutLag += spoutLag;
+ topicMetrics.totalEarliestTimeOffset += earliestTimeOffset;
+ topicMetrics.totalLatestTimeOffset += latestTimeOffset;
+ topicMetrics.totalLatestEmittedOffset += latestEmittedOffset;
+ topicMetrics.totalLatestCompletedOffset += latestCompletedOffset;
+ }
+
+ for (Map.Entry<String, TopicMetrics> e : topicMetricsMap.entrySet()) {
+ String topic = e.getKey();
+ TopicMetrics topicMetrics = e.getValue();
+ ret.put(topic + "/" + "totalSpoutLag", topicMetrics.totalSpoutLag);
+ ret.put(topic + "/" + "totalEarliestTimeOffset", topicMetrics.totalEarliestTimeOffset);
+ ret.put(topic + "/" + "totalLatestTimeOffset", topicMetrics.totalLatestTimeOffset);
+ ret.put(topic + "/" + "totalLatestEmittedOffset", topicMetrics.totalLatestEmittedOffset);
+ ret.put(topic + "/" + "totalLatestCompletedOffset", topicMetrics.totalLatestCompletedOffset);
+ }
+
+ return ret;
+ } else {
+ LOG.info("Metrics Tick: Not enough data to calculate spout lag.");
+ }
+ } catch (Throwable t) {
+ LOG.warn("Metrics Tick: Exception when computing kafkaOffset metric.", t);
+ }
+ return null;
+ }
+
+ public void refreshPartitions(Set<Partition> partitions) {
+ _partitions = partitions;
+ Iterator<Partition> it = _partitionToOffset.keySet().iterator();
+ while (it.hasNext()) {
+ if (!partitions.contains(it.next())) {
+ it.remove();
+ }
+ }
+ }
+
+ private class TopicMetrics {
+ long totalSpoutLag = 0;
+ long totalEarliestTimeOffset = 0;
+ long totalLatestTimeOffset = 0;
+ long totalLatestEmittedOffset = 0;
+ long totalLatestCompletedOffset = 0;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KeyValueScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KeyValueScheme.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KeyValueScheme.java
index 3f9acc2..6bb1dc5 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KeyValueScheme.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KeyValueScheme.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka;
-import org.apache.storm.spout.Scheme;
+package org.apache.storm.kafka;
import java.nio.ByteBuffer;
import java.util.List;
+import org.apache.storm.spout.Scheme;
public interface KeyValueScheme extends Scheme {
List<Object> deserializeKeyAndValue(ByteBuffer key, ByteBuffer value);
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KeyValueSchemeAsMultiScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KeyValueSchemeAsMultiScheme.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KeyValueSchemeAsMultiScheme.java
index 25053dd..00983cc 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/KeyValueSchemeAsMultiScheme.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/KeyValueSchemeAsMultiScheme.java
@@ -1,27 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka;
-import org.apache.storm.spout.SchemeAsMultiScheme;
+package org.apache.storm.kafka;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
+import org.apache.storm.spout.SchemeAsMultiScheme;
public class KeyValueSchemeAsMultiScheme extends SchemeAsMultiScheme {
@@ -30,9 +24,12 @@ public class KeyValueSchemeAsMultiScheme extends SchemeAsMultiScheme {
}
public Iterable<List<Object>> deserializeKeyAndValue(final ByteBuffer key, final ByteBuffer value) {
- List<Object> o = ((KeyValueScheme)scheme).deserializeKeyAndValue(key, value);
- if(o == null) return null;
- else return Arrays.asList(o);
+ List<Object> o = ((KeyValueScheme) scheme).deserializeKeyAndValue(key, value);
+ if (o == null) {
+ return null;
+ } else {
+ return Arrays.asList(o);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/MessageMetadataScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/MessageMetadataScheme.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/MessageMetadataScheme.java
index d0fc08e..f77f419 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/MessageMetadataScheme.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/MessageMetadataScheme.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka;
-import org.apache.storm.spout.Scheme;
+package org.apache.storm.kafka;
import java.nio.ByteBuffer;
import java.util.List;
+import org.apache.storm.spout.Scheme;
public interface MessageMetadataScheme extends Scheme {
List<Object> deserializeMessageWithMetadata(ByteBuffer message, Partition partition, long offset);
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/MessageMetadataSchemeAsMultiScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/MessageMetadataSchemeAsMultiScheme.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/MessageMetadataSchemeAsMultiScheme.java
index a53fa88..f52a772 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/MessageMetadataSchemeAsMultiScheme.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/MessageMetadataSchemeAsMultiScheme.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
-
import org.apache.storm.spout.SchemeAsMultiScheme;
public class MessageMetadataSchemeAsMultiScheme extends SchemeAsMultiScheme {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/Partition.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/Partition.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/Partition.java
index 99bb9d3..9edf28b 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/Partition.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/Partition.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import com.google.common.base.Objects;
@@ -33,16 +28,17 @@ public class Partition implements ISpoutPartition, Serializable {
// for kryo compatibility
private Partition() {
-
+
}
+
public Partition(Broker host, String topic, int partition) {
this.topic = topic;
this.host = host;
this.partition = partition;
this.bUseTopicNameForPartitionPathId = false;
}
-
- public Partition(Broker host, String topic, int partition,Boolean bUseTopicNameForPartitionPathId) {
+
+ public Partition(Broker host, String topic, int partition, Boolean bUseTopicNameForPartitionPathId) {
this.topic = topic;
this.host = host;
this.partition = partition;
@@ -63,22 +59,23 @@ public class Partition implements ISpoutPartition, Serializable {
return false;
}
final Partition other = (Partition) obj;
- return Objects.equal(this.host, other.host) && Objects.equal(this.topic, other.topic) && Objects.equal(this.partition, other.partition);
+ return Objects.equal(this.host, other.host) && Objects.equal(this.topic, other.topic) &&
+ Objects.equal(this.partition, other.partition);
}
@Override
public String toString() {
return "Partition{" +
- "host=" + host +
- ", topic=" + topic +
- ", partition=" + partition +
- '}';
+ "host=" + host +
+ ", topic=" + topic +
+ ", partition=" + partition +
+ '}';
}
@Override
public String getId() {
if (bUseTopicNameForPartitionPathId) {
- return topic + "/partition_" + partition;
+ return topic + "/partition_" + partition;
} else {
//Keep the Partition Id backward compatible with Old implementation of Partition.getId() == "partition_" + partition
return "partition_" + partition;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionCoordinator.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionCoordinator.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionCoordinator.java
index c9004fa..4dba709 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionCoordinator.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionCoordinator.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import java.util.List;
[12/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java
index 1521a96..b430f0b 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.localizer;
@@ -22,7 +16,6 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.net.JarURLConnection;
import java.net.URL;
import java.nio.file.DirectoryStream;
@@ -35,11 +28,9 @@ import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.storm.blobstore.ClientBlobStore;
-import org.apache.storm.blobstore.InputStreamWithMeta;
import org.apache.storm.daemon.supervisor.AdvancedFSOps;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.KeyNotFoundException;
@@ -54,69 +45,9 @@ import org.slf4j.LoggerFactory;
* The version number of the blob's file will be stored in `${basename}.version`
*/
public class LocallyCachedTopologyBlob extends LocallyCachedBlob {
- private static final Logger LOG = LoggerFactory.getLogger(LocallyCachedTopologyBlob.class);
public static final long LOCAL_MODE_JAR_VERSION = 1;
-
- private static String resourcesJar() throws IOException {
- String path = ServerUtils.currentClasspath();
- if (path == null) {
- return null;
- }
-
- for (String jpath : path.split(File.pathSeparator)) {
- if (jpath.endsWith(".jar")) {
- if (ServerUtils.zipDoesContainDir(jpath, ServerConfigUtils.RESOURCES_SUBDIR)) {
- return jpath;
- }
- }
- }
- return null;
- }
-
- public enum TopologyBlobType {
- TOPO_JAR("stormjar.jar", "-stormjar.jar", "resources"),
- TOPO_CODE("stormcode.ser", "-stormcode.ser", null),
- TOPO_CONF("stormconf.ser", "-stormconf.ser", null);
-
- private final String fileName;
- private final String keySuffix;
- private final String extractionDir;
-
- TopologyBlobType(String fileName, String keySuffix, String extractionDir) {
- this.fileName = fileName;
- this.keySuffix = keySuffix;
- this.extractionDir = extractionDir;
- }
-
- public String getFileName() {
- return fileName;
- }
-
- public String getTempFileName(long version) {
- return fileName + "." + version;
- }
-
- public String getVersionFileName() {
- return fileName + ".version";
- }
-
- public String getKey(String topologyId) {
- return topologyId + keySuffix;
- }
-
- public boolean needsExtraction() {
- return extractionDir != null;
- }
-
- public String getExtractionDir() {
- return extractionDir;
- }
-
- public String getTempExtractionDir(long version) {
- return extractionDir + "." + version;
- }
- }
-
+ private static final Logger LOG = LoggerFactory.getLogger(LocallyCachedTopologyBlob.class);
+ private static final Pattern EXTRACT_BASE_NAME_AND_VERSION = Pattern.compile("^(.*)\\.([0-9]+)$");
private final TopologyBlobType type;
private final String topologyId;
private final boolean isLocalMode;
@@ -124,7 +55,6 @@ public class LocallyCachedTopologyBlob extends LocallyCachedBlob {
private final AdvancedFSOps fsOps;
private volatile long version = NOT_DOWNLOADED_VERSION;
private volatile long size = 0;
-
/**
* Create a new LocallyCachedBlob.
*
@@ -143,6 +73,22 @@ public class LocallyCachedTopologyBlob extends LocallyCachedBlob {
updateSizeOnDisk();
}
+ private static String resourcesJar() throws IOException {
+ String path = ServerUtils.currentClasspath();
+ if (path == null) {
+ return null;
+ }
+
+ for (String jpath : path.split(File.pathSeparator)) {
+ if (jpath.endsWith(".jar")) {
+ if (ServerUtils.zipDoesContainDir(jpath, ServerConfigUtils.RESOURCES_SUBDIR)) {
+ return jpath;
+ }
+ }
+ }
+ return null;
+ }
+
private void updateSizeOnDisk() throws IOException {
long total = getSizeOnDisk(topologyBasicBlobsRootDir.resolve(type.getFileName()));
if (type.needsExtraction()) {
@@ -203,14 +149,14 @@ public class LocallyCachedTopologyBlob extends LocallyCachedBlob {
long newVersion = downloadToTempLocation(store, type.getKey(topologyId), version, fsOps,
- (version) -> topologyBasicBlobsRootDir.resolve(type.getTempFileName(version)));
+ (version) -> topologyBasicBlobsRootDir.resolve(type.getTempFileName(version)));
Path tmpLocation = topologyBasicBlobsRootDir.resolve(type.getTempFileName(newVersion));
if (type.needsExtraction()) {
Path extractionDest = topologyBasicBlobsRootDir.resolve(type.getTempExtractionDir(newVersion));
extractDirFromJar(tmpLocation.toAbsolutePath().toString(), ServerConfigUtils.RESOURCES_SUBDIR,
- extractionDest);
+ extractionDest);
}
return newVersion;
}
@@ -295,16 +241,14 @@ public class LocallyCachedTopologyBlob extends LocallyCachedBlob {
}
}
- private static final Pattern EXTRACT_BASE_NAME_AND_VERSION = Pattern.compile("^(.*)\\.([0-9]+)$");
-
private void cleanUpTemp(String baseName) throws IOException {
LOG.debug("Cleaning up temporary data in {}", topologyBasicBlobsRootDir);
try (DirectoryStream<Path> children = fsOps.newDirectoryStream(topologyBasicBlobsRootDir,
- (p) -> {
- String fileName = p.getFileName().toString();
- Matcher m = EXTRACT_BASE_NAME_AND_VERSION.matcher(fileName);
- return m.matches() && baseName.equals(m.group(1));
- })) {
+ (p) -> {
+ String fileName = p.getFileName().toString();
+ Matcher m = EXTRACT_BASE_NAME_AND_VERSION.matcher(fileName);
+ return m.matches() && baseName.equals(m.group(1));
+ })) {
//children is only ever null if topologyBasicBlobsRootDir does not exist. This happens during unit tests
// And because a non-existant directory is by definition clean we are ignoring it.
if (children != null) {
@@ -344,7 +288,7 @@ public class LocallyCachedTopologyBlob extends LocallyCachedBlob {
@Override
public boolean equals(Object other) {
if (other instanceof LocallyCachedTopologyBlob) {
- LocallyCachedTopologyBlob o = (LocallyCachedTopologyBlob)other;
+ LocallyCachedTopologyBlob o = (LocallyCachedTopologyBlob) other;
return topologyId.equals(o.topologyId) && type == o.type && topologyBasicBlobsRootDir.equals(o.topologyBasicBlobsRootDir);
}
return false;
@@ -359,4 +303,48 @@ public class LocallyCachedTopologyBlob extends LocallyCachedBlob {
public String toString() {
return "LOCAL TOPO BLOB " + type + " " + topologyId;
}
+
+ public enum TopologyBlobType {
+ TOPO_JAR("stormjar.jar", "-stormjar.jar", "resources"),
+ TOPO_CODE("stormcode.ser", "-stormcode.ser", null),
+ TOPO_CONF("stormconf.ser", "-stormconf.ser", null);
+
+ private final String fileName;
+ private final String keySuffix;
+ private final String extractionDir;
+
+ TopologyBlobType(String fileName, String keySuffix, String extractionDir) {
+ this.fileName = fileName;
+ this.keySuffix = keySuffix;
+ this.extractionDir = extractionDir;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public String getTempFileName(long version) {
+ return fileName + "." + version;
+ }
+
+ public String getVersionFileName() {
+ return fileName + ".version";
+ }
+
+ public String getKey(String topologyId) {
+ return topologyId + keySuffix;
+ }
+
+ public boolean needsExtraction() {
+ return extractionDir != null;
+ }
+
+ public String getExtractionDir() {
+ return extractionDir;
+ }
+
+ public String getTempExtractionDir(long version) {
+ return extractionDir + "." + version;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java b/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java
index bd92173..0979e03 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.localizer;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java b/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java
index e326174..78ab822 100644
--- a/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java
+++ b/storm-server/src/main/java/org/apache/storm/logging/ThriftAccessLogger.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.logging;
@@ -29,7 +23,7 @@ public class ThriftAccessLogger {
private static String accessLogBase = "Request ID: {} access from: {} principal: {} operation: {}";
public static void logAccessFunction(Integer requestId, InetAddress remoteAddress,
- Principal principal, String operation,
+ Principal principal, String operation,
String function) {
String functionPart = "";
if (function != null) {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/logging/filters/AccessLoggingFilter.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/logging/filters/AccessLoggingFilter.java b/storm-server/src/main/java/org/apache/storm/logging/filters/AccessLoggingFilter.java
index de4cb72..5d7df0c 100644
--- a/storm-server/src/main/java/org/apache/storm/logging/filters/AccessLoggingFilter.java
+++ b/storm-server/src/main/java/org/apache/storm/logging/filters/AccessLoggingFilter.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.logging.filters;
@@ -27,7 +21,6 @@ import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,7 +39,7 @@ public class AccessLoggingFilter implements Filter {
public void handle(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException {
if (request != null) {
LOG.info("Access from: {} url: {} principal: {}", request.getRemoteAddr(), request.getRequestURL(),
- (request.getUserPrincipal() == null ? "" : request.getUserPrincipal().getName()));
+ (request.getUserPrincipal() == null ? "" : request.getUserPrincipal().getName()));
}
chain.doFilter(request, response);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metric/ClusterMetricsConsumerExecutor.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metric/ClusterMetricsConsumerExecutor.java b/storm-server/src/main/java/org/apache/storm/metric/ClusterMetricsConsumerExecutor.java
index 2ee8db5..cfeb63c 100644
--- a/storm-server/src/main/java/org/apache/storm/metric/ClusterMetricsConsumerExecutor.java
+++ b/storm-server/src/main/java/org/apache/storm/metric/ClusterMetricsConsumerExecutor.java
@@ -1,35 +1,29 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.metric;
+import java.util.Collection;
import org.apache.storm.metric.api.DataPoint;
import org.apache.storm.metric.api.IClusterMetricsConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
-
public class ClusterMetricsConsumerExecutor {
public static final Logger LOG = LoggerFactory.getLogger(ClusterMetricsConsumerExecutor.class);
private static final String ERROR_MESSAGE_PREPARATION_CLUSTER_METRICS_CONSUMER_FAILED =
- "Preparation of Cluster Metrics Consumer failed. " +
- "Please check your configuration and/or corresponding systems and relaunch Nimbus. " +
- "Skipping handle metrics.";
+ "Preparation of Cluster Metrics Consumer failed. " +
+ "Please check your configuration and/or corresponding systems and relaunch Nimbus. " +
+ "Skipping handle metrics.";
private IClusterMetricsConsumer metricsConsumer;
private String consumerClassName;
@@ -42,11 +36,11 @@ public class ClusterMetricsConsumerExecutor {
public void prepare() {
try {
- metricsConsumer = (IClusterMetricsConsumer)Class.forName(consumerClassName).newInstance();
+ metricsConsumer = (IClusterMetricsConsumer) Class.forName(consumerClassName).newInstance();
metricsConsumer.prepare(registrationArgument);
} catch (Exception e) {
LOG.error("Could not instantiate or prepare Cluster Metrics Consumer with fully qualified name " +
- consumerClassName, e);
+ consumerClassName, e);
if (metricsConsumer != null) {
metricsConsumer.cleanup();
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metric/LoggingClusterMetricsConsumer.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metric/LoggingClusterMetricsConsumer.java b/storm-server/src/main/java/org/apache/storm/metric/LoggingClusterMetricsConsumer.java
index db47d84..f07dc54 100644
--- a/storm-server/src/main/java/org/apache/storm/metric/LoggingClusterMetricsConsumer.java
+++ b/storm-server/src/main/java/org/apache/storm/metric/LoggingClusterMetricsConsumer.java
@@ -1,29 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.metric;
+import java.util.Collection;
import org.apache.storm.metric.api.DataPoint;
import org.apache.storm.metric.api.IClusterMetricsConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
-
/**
* Listens for cluster related metrics, dumps them to log
*
@@ -48,7 +42,7 @@ public class LoggingClusterMetricsConsumer implements IClusterMetricsConsumer {
public void handleDataPoints(ClusterInfo clusterInfo, Collection<DataPoint> dataPoints) {
StringBuilder sb = new StringBuilder();
String header = String.format("%d\t%15s\t%40s\t",
- clusterInfo.getTimestamp(), "<cluster>", "<cluster>");
+ clusterInfo.getTimestamp(), "<cluster>", "<cluster>");
sb.append(header);
logDataPoints(dataPoints, sb, header);
}
@@ -57,15 +51,15 @@ public class LoggingClusterMetricsConsumer implements IClusterMetricsConsumer {
public void handleDataPoints(SupervisorInfo supervisorInfo, Collection<DataPoint> dataPoints) {
StringBuilder sb = new StringBuilder();
String header = String.format("%d\t%15s\t%40s\t",
- supervisorInfo.getTimestamp(),
- supervisorInfo.getSrcSupervisorHost(),
- supervisorInfo.getSrcSupervisorId());
+ supervisorInfo.getTimestamp(),
+ supervisorInfo.getSrcSupervisorHost(),
+ supervisorInfo.getSrcSupervisorId());
sb.append(header);
for (DataPoint p : dataPoints) {
sb.delete(header.length(), sb.length());
sb.append(p.getName())
- .append(padding).delete(header.length()+23,sb.length()).append("\t")
- .append(p.getValue());
+ .append(padding).delete(header.length() + 23, sb.length()).append("\t")
+ .append(p.getValue());
LOG.info(sb.toString());
}
}
@@ -78,8 +72,8 @@ public class LoggingClusterMetricsConsumer implements IClusterMetricsConsumer {
for (DataPoint p : dataPoints) {
sb.delete(header.length(), sb.length());
sb.append(p.getName())
- .append(padding).delete(header.length()+23,sb.length()).append("\t")
- .append(p.getValue());
+ .append(padding).delete(header.length() + 23, sb.length()).append("\t")
+ .append(p.getValue());
LOG.info(sb.toString());
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metric/StormMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metric/StormMetricsRegistry.java b/storm-server/src/main/java/org/apache/storm/metric/StormMetricsRegistry.java
index 28bba3e..3bc50e7 100644
--- a/storm-server/src/main/java/org/apache/storm/metric/StormMetricsRegistry.java
+++ b/storm-server/src/main/java/org/apache/storm/metric/StormMetricsRegistry.java
@@ -1,28 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metric;
-import com.codahale.metrics.*;
-
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Reservoir;
import java.util.Map;
import java.util.concurrent.Callable;
-
import org.apache.storm.daemon.metrics.MetricsUtils;
import org.apache.storm.daemon.metrics.reporters.PreparableReporter;
import org.slf4j.Logger;
@@ -30,8 +27,8 @@ import org.slf4j.LoggerFactory;
@SuppressWarnings("unchecked")
public class StormMetricsRegistry {
- private static final Logger LOG = LoggerFactory.getLogger(StormMetricsRegistry.class);
public static final MetricRegistry DEFAULT_REGISTRY = new MetricRegistry();
+ private static final Logger LOG = LoggerFactory.getLogger(StormMetricsRegistry.class);
public static Meter registerMeter(String name) {
Meter meter = new Meter();
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metric/api/DataPoint.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metric/api/DataPoint.java b/storm-server/src/main/java/org/apache/storm/metric/api/DataPoint.java
index 615dd5b..8468722 100644
--- a/storm-server/src/main/java/org/apache/storm/metric/api/DataPoint.java
+++ b/storm-server/src/main/java/org/apache/storm/metric/api/DataPoint.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.metric.api;
public class DataPoint {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metric/api/IClusterMetricsConsumer.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metric/api/IClusterMetricsConsumer.java b/storm-server/src/main/java/org/apache/storm/metric/api/IClusterMetricsConsumer.java
index 39d60f3..a0b0a8d 100644
--- a/storm-server/src/main/java/org/apache/storm/metric/api/IClusterMetricsConsumer.java
+++ b/storm-server/src/main/java/org/apache/storm/metric/api/IClusterMetricsConsumer.java
@@ -1,25 +1,28 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.metric.api;
import java.util.Collection;
public interface IClusterMetricsConsumer {
+ void prepare(Object registrationArgument);
+
+ void handleDataPoints(ClusterInfo clusterInfo, Collection<DataPoint> dataPoints);
+
+ void handleDataPoints(SupervisorInfo supervisorInfo, Collection<DataPoint> dataPoints);
+
+ void cleanup();
+
class ClusterInfo {
private long timestamp;
@@ -55,9 +58,4 @@ public interface IClusterMetricsConsumer {
return timestamp;
}
}
-
- void prepare(Object registrationArgument);
- void handleDataPoints(ClusterInfo clusterInfo, Collection<DataPoint> dataPoints);
- void handleDataPoints(SupervisorInfo supervisorInfo, Collection<DataPoint> dataPoints);
- void cleanup();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/AggLevel.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/AggLevel.java b/storm-server/src/main/java/org/apache/storm/metricstore/AggLevel.java
index 662a17c..d8beaa0 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/AggLevel.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/AggLevel.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore;
@@ -30,7 +23,7 @@ public enum AggLevel {
private final byte value;
AggLevel(int value) {
- this.value = (byte)value;
+ this.value = (byte) value;
}
public byte getValue() {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/FilterOptions.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/FilterOptions.java b/storm-server/src/main/java/org/apache/storm/metricstore/FilterOptions.java
index 7cfbfbe..33d2097 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/FilterOptions.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/FilterOptions.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore;
@@ -39,32 +32,28 @@ public class FilterOptions {
public FilterOptions() {
}
- public void setTopologyId(String topoId) {
- this.topologyId = topoId;
- }
-
public String getTopologyId() {
return this.topologyId;
}
- public void setComponentId(String component) {
- this.componentId = component;
+ public void setTopologyId(String topoId) {
+ this.topologyId = topoId;
}
public String getComponentId() {
return this.componentId;
}
- public void setStartTime(Long time) {
- this.startTime = time;
+ public void setComponentId(String component) {
+ this.componentId = component;
}
public long getStartTime() {
return this.startTime;
}
- public void setEndTime(Long time) {
- this.endTime = time;
+ public void setStartTime(Long time) {
+ this.startTime = time;
}
/**
@@ -77,46 +66,50 @@ public class FilterOptions {
return this.endTime;
}
- public void setMetricName(String name) {
- this.metricName = name;
+ public void setEndTime(Long time) {
+ this.endTime = time;
}
public String getMetricName() {
return this.metricName;
}
- public void setExecutorId(String id) {
- this.executorId = id;
+ public void setMetricName(String name) {
+ this.metricName = name;
}
public String getExecutorId() {
return this.executorId;
}
- public void setHostId(String id) {
- this.hostId = id;
+ public void setExecutorId(String id) {
+ this.executorId = id;
}
public String getHostId() {
return this.hostId;
}
- public void setPort(Integer p) {
- this.port = p;
+ public void setHostId(String id) {
+ this.hostId = id;
}
public Integer getPort() {
return this.port;
}
- public void setStreamId(String id) {
- this.streamId = id;
+ public void setPort(Integer p) {
+ this.port = p;
}
public String getStreamId() {
return this.streamId;
}
+ public void setStreamId(String id) {
+ this.streamId = id;
+ }
+
/**
* Add an aggregation level to search for.
*/
@@ -128,16 +121,6 @@ public class FilterOptions {
}
/**
- * Set the aggregation levels to search for.
- */
- public void setAggLevels(Set<AggLevel> levels) throws MetricException {
- this.aggLevels = levels;
- if (this.aggLevels == null || this.aggLevels.isEmpty()) {
- throw new MetricException("Cannot search for empty AggLevel");
- }
- }
-
- /**
* Get the aggregation levels to search for.
*/
public Set<AggLevel> getAggLevels() {
@@ -151,4 +134,14 @@ public class FilterOptions {
}
return this.aggLevels;
}
+
+ /**
+ * Set the aggregation levels to search for.
+ */
+ public void setAggLevels(Set<AggLevel> levels) throws MetricException {
+ this.aggLevels = levels;
+ if (this.aggLevels == null || this.aggLevels.isEmpty()) {
+ throw new MetricException("Cannot search for empty AggLevel");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/Metric.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/Metric.java b/storm-server/src/main/java/org/apache/storm/metricstore/Metric.java
index 716ced0..19de781 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/Metric.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/Metric.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore;
@@ -22,7 +15,6 @@ import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,20 +92,20 @@ public class Metric implements Comparable<Metric> {
Metric o = (Metric) other;
return this == other
- || (this.name.equals(o.getMetricName())
- && this.timestamp == o.getTimestamp()
- && this.topologyId.equals(o.getTopologyId())
- && this.value == o.getValue()
- && this.componentId.equals(o.getComponentId())
- && this.executorId.equals(o.getExecutorId())
- && this.hostname.equals(o.getHostname())
- && this.streamId.equals(o.getStreamId())
- && this.port == o.getPort()
- && this.count == o.getCount()
- && this.min == o.getMin()
- && this.max == o.getMax()
- && this.sum == o.getSum()
- && this.aggLevel == o.getAggLevel());
+ || (this.name.equals(o.getMetricName())
+ && this.timestamp == o.getTimestamp()
+ && this.topologyId.equals(o.getTopologyId())
+ && this.value == o.getValue()
+ && this.componentId.equals(o.getComponentId())
+ && this.executorId.equals(o.getExecutorId())
+ && this.hostname.equals(o.getHostname())
+ && this.streamId.equals(o.getStreamId())
+ && this.port == o.getPort()
+ && this.count == o.getCount()
+ && this.min == o.getMin()
+ && this.max == o.getMax()
+ && this.sum == o.getSum()
+ && this.aggLevel == o.getAggLevel());
}
public AggLevel getAggLevel() {
@@ -131,17 +123,6 @@ public class Metric implements Comparable<Metric> {
}
/**
- * Initialize the metric value.
- */
- public void setValue(double value) {
- this.count = 1L;
- this.min = value;
- this.max = value;
- this.sum = value;
- this.value = value;
- }
-
- /**
* Adds an additional value to the metric.
*/
public void addValue(double value) {
@@ -204,6 +185,17 @@ public class Metric implements Comparable<Metric> {
return this.value;
}
+ /**
+ * Initialize the metric value.
+ */
+ public void setValue(double value) {
+ this.count = 1L;
+ this.min = value;
+ this.max = value;
+ this.sum = value;
+ this.value = value;
+ }
+
public String getMetricName() {
return this.name;
}
@@ -259,12 +251,12 @@ public class Metric implements Comparable<Metric> {
sb.append("|");
sb.append(this.streamId);
return String.format("%s -- count: %d -- value: %f -- min: %f -- max: %f -- sum: %f",
- sb.toString(),
- this.count,
- this.value,
- this.min,
- this.max,
- this.sum);
+ sb.toString(),
+ this.count,
+ this.value,
+ this.min,
+ this.max,
+ this.sum);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/MetricException.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/MetricException.java b/storm-server/src/main/java/org/apache/storm/metricstore/MetricException.java
index e45a451..8236e7f 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/MetricException.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/MetricException.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/MetricStore.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/MetricStore.java b/storm-server/src/main/java/org/apache/storm/metricstore/MetricStore.java
index d43e11b..71f53a5 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/MetricStore.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/MetricStore.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore;
@@ -42,7 +35,7 @@ public interface MetricStore extends AutoCloseable {
* Fill out the numeric values for a metric.
*
* @param metric Metric to populate
- * @return true if the metric was populated, false otherwise
+ * @return true if the metric was populated, false otherwise
* @throws MetricException on error
*/
boolean populateValue(Metric metric) throws MetricException;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/MetricStoreConfig.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/MetricStoreConfig.java b/storm-server/src/main/java/org/apache/storm/metricstore/MetricStoreConfig.java
index d66529a..42e5a7d 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/MetricStoreConfig.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/MetricStoreConfig.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore;
@@ -33,7 +26,7 @@ public class MetricStoreConfig {
public static MetricStore configure(Map<String, Object> conf) throws MetricException {
try {
- String storeClass = (String)conf.get(DaemonConfig.STORM_METRIC_STORE_CLASS);
+ String storeClass = (String) conf.get(DaemonConfig.STORM_METRIC_STORE_CLASS);
MetricStore store = (MetricStore) (Class.forName(storeClass)).newInstance();
store.prepare(conf);
return store;
@@ -51,7 +44,7 @@ public class MetricStoreConfig {
public static WorkerMetricsProcessor configureMetricProcessor(Map conf) throws MetricException {
try {
- String processorClass = (String)conf.get(DaemonConfig.STORM_METRIC_PROCESSOR_CLASS);
+ String processorClass = (String) conf.get(DaemonConfig.STORM_METRIC_PROCESSOR_CLASS);
WorkerMetricsProcessor processor = (WorkerMetricsProcessor) (Class.forName(processorClass)).newInstance();
processor.prepare(conf);
return processor;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/NimbusMetricProcessor.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/NimbusMetricProcessor.java b/storm-server/src/main/java/org/apache/storm/metricstore/NimbusMetricProcessor.java
index 0c98208..f3970b8 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/NimbusMetricProcessor.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/NimbusMetricProcessor.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore;
@@ -26,7 +19,7 @@ import org.apache.thrift.TException;
/**
* Implementation of WorkerMetricsProcessor that sends metric data to Nimbus for processing.
*/
-public class NimbusMetricProcessor implements WorkerMetricsProcessor {
+public class NimbusMetricProcessor implements WorkerMetricsProcessor {
@Override
public void processWorkerMetrics(Map<String, Object> conf, WorkerMetrics metrics) throws MetricException {
try (NimbusClient client = NimbusClient.getConfiguredClient(conf)) {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/WorkerMetricsProcessor.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/WorkerMetricsProcessor.java b/storm-server/src/main/java/org/apache/storm/metricstore/WorkerMetricsProcessor.java
index 98c1163..5a59473 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/WorkerMetricsProcessor.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/WorkerMetricsProcessor.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/KeyType.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/KeyType.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/KeyType.java
index a351be7..8f65591 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/KeyType.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/KeyType.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore.rocksdb;
@@ -38,7 +31,6 @@ public enum KeyType {
METADATA_STRING_END(7),
METRIC_DATA(0x80);
- private final byte value;
private static Map<Byte, KeyType> MAP;
static {
@@ -49,12 +41,10 @@ public enum KeyType {
MAP = Collections.unmodifiableMap(MAP);
}
- KeyType(int value) {
- this.value = (byte)value;
- }
+ private final byte value;
- byte getValue() {
- return this.value;
+ KeyType(int value) {
+ this.value = (byte) value;
}
static KeyType getKeyType(byte value) {
@@ -65,6 +55,10 @@ public enum KeyType {
return type;
}
}
+
+ byte getValue() {
+ return this.value;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/MetricsCleaner.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/MetricsCleaner.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/MetricsCleaner.java
index 6618f5d..07fe862 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/MetricsCleaner.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/MetricsCleaner.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore.rocksdb;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/ReadOnlyStringMetadataCache.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/ReadOnlyStringMetadataCache.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/ReadOnlyStringMetadataCache.java
index 0effbc4..11d82b7 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/ReadOnlyStringMetadataCache.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/ReadOnlyStringMetadataCache.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore.rocksdb;
@@ -30,7 +23,7 @@ public interface ReadOnlyStringMetadataCache {
* Get the string metadata from the cache.
*
* @param s The string to look for
- * @return the metadata associated with the string or null if not found
+ * @return the metadata associated with the string or null if not found
*/
StringMetadata get(String s);
@@ -38,7 +31,7 @@ public interface ReadOnlyStringMetadataCache {
* Returns the string matching the string Id if in the cache.
*
* @param stringId The string Id to check
- * @return the associated string if the Id is in the cache, null otherwise
+ * @return the associated string if the Id is in the cache, null otherwise
*/
String getMetadataString(Integer stringId);
@@ -46,7 +39,7 @@ public interface ReadOnlyStringMetadataCache {
* Determines if a string Id is contained in the cache.
*
* @param stringId The string Id to check
- * @return true if the Id is in the cache, false otherwise
+ * @return true if the Id is in the cache, false otherwise
*/
boolean contains(Integer stringId);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbKey.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbKey.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbKey.java
index 7868282..fffcaf1 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbKey.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbKey.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore.rocksdb;
@@ -51,10 +44,9 @@ import org.slf4j.LoggerFactory;
* </pre>
*/
public class RocksDbKey implements Comparable<RocksDbKey> {
- private static final Logger LOG = LoggerFactory.getLogger(RocksDbKey.class);
static final int KEY_SIZE = 38;
+ private static final Logger LOG = LoggerFactory.getLogger(RocksDbKey.class);
private static Map<Byte, RocksDbKey> PREFIX_MAP = new HashMap<>();
- private byte[] key;
static {
// pregenerate commonly used keys for scans
@@ -65,6 +57,8 @@ public class RocksDbKey implements Comparable<RocksDbKey> {
PREFIX_MAP = Collections.unmodifiableMap(PREFIX_MAP);
}
+ private byte[] key;
+
/**
* Constructor for a RocksDB key for a metadata string.
*
@@ -94,56 +88,16 @@ public class RocksDbKey implements Comparable<RocksDbKey> {
* Get a zeroed key of the specified type.
*
* @param type the desired type
- * @return a key of the desired type
+ * @return a key of the desired type
*/
static RocksDbKey getPrefix(KeyType type) {
return PREFIX_MAP.get(type.getValue());
}
/**
- * get the metadata string Id portion of the key for metadata keys.
- *
- * @return the metadata string Id
- * @throws RuntimeException if the key is not a metadata type
- */
- int getMetadataStringId() {
- if (this.getType().getValue() < KeyType.METADATA_STRING_END.getValue()) {
- return ByteBuffer.wrap(key, 2, 4).getInt();
- } else {
- throw new RuntimeException("Cannot fetch metadata string for key of type " + this.getType());
- }
- }
-
- /**
- * get the raw key bytes
- */
- byte[] getRaw() {
- return this.key;
- }
-
- /**
- * get the type of key.
- *
- * @return the type of key
- */
- KeyType getType() {
- return KeyType.getKeyType(key[0]);
- }
-
- /**
- * compares to keys on a byte by byte basis.
- *
- * @return comparison of key byte values
- */
- @Override
- public int compareTo(RocksDbKey o) {
- return UnsignedBytes.lexicographicalComparator().compare(this.getRaw(), o.getRaw());
- }
-
- /**
* gets the first possible key value for the desired key type.
*
- * @return the initial key
+ * @return the initial key
*/
static RocksDbKey getInitialKey(KeyType type) {
return PREFIX_MAP.get(type.getValue());
@@ -152,17 +106,17 @@ public class RocksDbKey implements Comparable<RocksDbKey> {
/**
* gets the key just larger than the last possible key value for the desired key type.
*
- * @return the last key
+ * @return the last key
*/
static RocksDbKey getLastKey(KeyType type) {
- byte value = (byte)(type.getValue() + 1);
+ byte value = (byte) (type.getValue() + 1);
return PREFIX_MAP.get(value);
}
/**
* Creates a metric key with the desired properties.
*
- * @return the generated key
+ * @return the generated key
*/
static RocksDbKey createMetricKey(AggLevel aggLevel, int topologyId, long metricTimestamp, int metricId,
int componentId, int executorId, int hostId, int port,
@@ -185,6 +139,46 @@ public class RocksDbKey implements Comparable<RocksDbKey> {
}
/**
+ * get the metadata string Id portion of the key for metadata keys.
+ *
+ * @return the metadata string Id
+ * @throws RuntimeException if the key is not a metadata type
+ */
+ int getMetadataStringId() {
+ if (this.getType().getValue() < KeyType.METADATA_STRING_END.getValue()) {
+ return ByteBuffer.wrap(key, 2, 4).getInt();
+ } else {
+ throw new RuntimeException("Cannot fetch metadata string for key of type " + this.getType());
+ }
+ }
+
+ /**
+ * get the raw key bytes
+ */
+ byte[] getRaw() {
+ return this.key;
+ }
+
+ /**
+ * get the type of key.
+ *
+ * @return the type of key
+ */
+ KeyType getType() {
+ return KeyType.getKeyType(key[0]);
+ }
+
+ /**
+ * compares to keys on a byte by byte basis.
+ *
+ * @return comparison of key byte values
+ */
+ @Override
+ public int compareTo(RocksDbKey o) {
+ return UnsignedBytes.lexicographicalComparator().compare(this.getRaw(), o.getRaw());
+ }
+
+ /**
* Get the unique string Id for a metric's topologyId.
*/
int getTopologyId() {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbMetricsWriter.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbMetricsWriter.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbMetricsWriter.java
index a050a76..7a30bcc 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbMetricsWriter.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbMetricsWriter.java
@@ -1,25 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore.rocksdb;
import com.codahale.metrics.Meter;
-
import java.util.ArrayList;
import java.util.HashSet;
import java.util.ListIterator;
@@ -28,7 +20,6 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ThreadLocalRandom;
-
import org.apache.http.annotation.NotThreadSafe;
import org.apache.storm.metricstore.AggLevel;
import org.apache.storm.metricstore.Metric;
@@ -79,7 +70,7 @@ public class RocksDbMetricsWriter implements Runnable, AutoCloseable {
* @param store The RocksDB store
* @param queue The queue to receive metrics for insertion
*/
- RocksDbMetricsWriter(RocksDbStore store, BlockingQueue queue, Meter failureMeter) {
+ RocksDbMetricsWriter(RocksDbStore store, BlockingQueue queue, Meter failureMeter) {
this.store = store;
this.queue = queue;
this.failureMeter = failureMeter;
@@ -134,7 +125,7 @@ public class RocksDbMetricsWriter implements Runnable, AutoCloseable {
Integer streamId = storeMetadataString(KeyType.STREAM_ID_STRING, metric.getStreamId(), metricTimestamp);
RocksDbKey key = RocksDbKey.createMetricKey(AggLevel.AGG_LEVEL_NONE, topologyId, metric.getTimestamp(), metricId,
- componentId, executorId, hostId, metric.getPort(), streamId);
+ componentId, executorId, hostId, metric.getPort(), streamId);
// save metric key/value to be batched
RocksDbValue value = new RocksDbValue(metric);
@@ -146,7 +137,7 @@ public class RocksDbMetricsWriter implements Runnable, AutoCloseable {
ListIterator li = aggBuckets.listIterator(aggBuckets.size());
boolean populate = true;
while (li.hasPrevious()) {
- AggLevel bucket = (AggLevel)li.previous();
+ AggLevel bucket = (AggLevel) li.previous();
Metric aggMetric = new Metric(metric);
aggMetric.setAggLevel(bucket);
@@ -155,7 +146,7 @@ public class RocksDbMetricsWriter implements Runnable, AutoCloseable {
aggMetric.setTimestamp(roundedToBucket);
RocksDbKey aggKey = RocksDbKey.createMetricKey(bucket, topologyId, aggMetric.getTimestamp(), metricId,
- componentId, executorId, hostId, aggMetric.getPort(), streamId);
+ componentId, executorId, hostId, aggMetric.getPort(), streamId);
if (populate) {
// retrieve any existing aggregation matching this one and update the values
@@ -217,7 +208,7 @@ public class RocksDbMetricsWriter implements Runnable, AutoCloseable {
generateUniqueStringIds();
int id = unusedIds.iterator().next();
unusedIds.remove(id);
- return id;
+ return id;
}
// guarantees a list of unused string Ids exists. Once the list is empty, creates a new list
@@ -290,8 +281,8 @@ public class RocksDbMetricsWriter implements Runnable, AutoCloseable {
// get all metadata from the cache to put into the database
TreeMap<RocksDbKey, RocksDbValue> batchMap = new TreeMap<>(); // use a new map to prevent threading issues with writer thread
for (Map.Entry entry : stringMetadataCache.entrySet()) {
- String metadataString = (String)entry.getKey();
- StringMetadata val = (StringMetadata)entry.getValue();
+ String metadataString = (String) entry.getKey();
+ StringMetadata val = (StringMetadata) entry.getValue();
RocksDbValue rval = new RocksDbValue(val.getLastTimestamp(), metadataString);
for (KeyType type : val.getMetadataTypes()) { // save the metadata for all types of strings it matches
[50/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-hbase
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-hbase
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/880d14f1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/880d14f1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/880d14f1
Branch: refs/heads/master
Commit: 880d14f1e7c6d450375b195f3aa5bc4045151fab
Parents: 6ccf6a0
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 23:08:31 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 00:37:54 2018 -0400
----------------------------------------------------------------------
external/storm-hbase/pom.xml | 2 +-
.../storm/hbase/bolt/AbstractHBaseBolt.java | 42 ++---
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 42 ++---
.../storm/hbase/bolt/HBaseLookupBolt.java | 161 +++++++++---------
.../storm/hbase/bolt/mapper/HBaseMapper.java | 27 ++-
.../bolt/mapper/HBaseProjectionCriteria.java | 75 ++++-----
.../hbase/bolt/mapper/HBaseValueMapper.java | 31 ++--
.../hbase/bolt/mapper/SimpleHBaseMapper.java | 55 +++---
.../apache/storm/hbase/common/ColumnList.java | 166 +++++++++----------
.../apache/storm/hbase/common/HBaseClient.java | 42 ++---
.../org/apache/storm/hbase/common/IColumn.java | 26 ++-
.../org/apache/storm/hbase/common/ICounter.java | 25 ++-
.../org/apache/storm/hbase/common/Utils.java | 83 +++++-----
.../storm/hbase/state/HBaseKeyValueState.java | 36 ++--
.../hbase/state/HBaseKeyValueStateIterator.java | 42 ++---
.../hbase/state/HBaseKeyValueStateProvider.java | 31 ++--
.../mapper/SimpleTridentHBaseMapMapper.java | 2 +-
.../mapper/SimpleTridentHBaseMapper.java | 45 +++--
.../trident/mapper/TridentHBaseMapper.java | 34 ++--
.../hbase/trident/state/HBaseMapState.java | 142 ++++++++--------
.../storm/hbase/trident/state/HBaseQuery.java | 24 +--
.../storm/hbase/trident/state/HBaseState.java | 125 +++++++-------
.../hbase/trident/state/HBaseStateFactory.java | 24 +--
.../storm/hbase/trident/state/HBaseUpdater.java | 24 +--
.../trident/windowing/HBaseWindowsStore.java | 65 ++++----
.../windowing/HBaseWindowsStoreFactory.java | 24 +--
.../storm/hbase/state/HBaseClientTestUtil.java | 38 ++---
.../state/HBaseKeyValueStateIteratorTest.java | 28 ++--
.../state/HBaseKeyValueStateProviderTest.java | 15 +-
.../hbase/state/HBaseKeyValueStateTest.java | 35 ++--
30 files changed, 684 insertions(+), 827 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-hbase/pom.xml b/external/storm-hbase/pom.xml
index 6d96620..e32f3c3 100644
--- a/external/storm-hbase/pom.xml
+++ b/external/storm-hbase/pom.xml
@@ -131,7 +131,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>371</maxAllowedViolations>
+ <maxAllowedViolations>100</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/AbstractHBaseBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/AbstractHBaseBolt.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/AbstractHBaseBolt.java
index 9cf1935..0ff60b3 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/AbstractHBaseBolt.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/AbstractHBaseBolt.java
@@ -1,38 +1,32 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.bolt;
-import org.apache.storm.Config;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.base.BaseRichBolt;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.commons.lang.Validate;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.storm.Config;
import org.apache.storm.hbase.bolt.mapper.HBaseMapper;
import org.apache.storm.hbase.common.HBaseClient;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.base.BaseRichBolt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
// TODO support more configuration options, for now we're defaulting to the hbase-*.xml files found on the classpath
public abstract class AbstractHBaseBolt extends BaseRichBolt {
private static final Logger LOG = LoggerFactory.getLogger(AbstractHBaseBolt.class);
@@ -55,15 +49,15 @@ public abstract class AbstractHBaseBolt extends BaseRichBolt {
this.collector = collector;
final Configuration hbConfig = HBaseConfiguration.create();
- Map<String, Object> conf = (Map<String, Object>)topoConf.get(this.configKey);
- if(conf == null) {
+ Map<String, Object> conf = (Map<String, Object>) topoConf.get(this.configKey);
+ if (conf == null) {
throw new IllegalArgumentException("HBase configuration not found using key '" + this.configKey + "'");
}
- if(conf.get("hbase.rootdir") == null) {
+ if (conf.get("hbase.rootdir") == null) {
LOG.warn("No 'hbase.rootdir' value found in configuration! Using HBase defaults.");
}
- for(String key : conf.keySet()) {
+ for (String key : conf.keySet()) {
hbConfig.set(key, String.valueOf(conf.get(key)));
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/HBaseBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/HBaseBolt.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/HBaseBolt.java
index bae7aeb..fa6acb5 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/HBaseBolt.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/HBaseBolt.java
@@ -1,46 +1,39 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.bolt;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.storm.hbase.bolt.mapper.HBaseMapper;
+import org.apache.storm.hbase.common.ColumnList;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.utils.BatchHelper;
import org.apache.storm.utils.TupleUtils;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.storm.hbase.bolt.mapper.HBaseMapper;
-import org.apache.storm.hbase.common.ColumnList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
-import java.util.List;
-import java.util.LinkedList;
-
/**
* Basic bolt for writing to HBase.
*
* Note: Each HBaseBolt defined in a topology is tied to a specific table.
- *
*/
-public class HBaseBolt extends AbstractHBaseBolt {
+public class HBaseBolt extends AbstractHBaseBolt {
private static final Logger LOG = LoggerFactory.getLogger(HBaseBolt.class);
private static final int DEFAULT_FLUSH_INTERVAL_SECS = 1;
@@ -87,7 +80,8 @@ public class HBaseBolt extends AbstractHBaseBolt {
if (batchHelper.shouldHandle(tuple)) {
byte[] rowKey = this.mapper.rowKey(tuple);
ColumnList cols = this.mapper.columns(tuple);
- List<Mutation> mutations = hBaseClient.constructMutationReq(rowKey, cols, writeToWAL? Durability.SYNC_WAL : Durability.SKIP_WAL);
+ List<Mutation> mutations =
+ hBaseClient.constructMutationReq(rowKey, cols, writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL);
batchMutations.addAll(mutations);
batchHelper.addBatch(tuple);
}
@@ -98,7 +92,7 @@ public class HBaseBolt extends AbstractHBaseBolt {
batchHelper.ack();
batchMutations.clear();
}
- } catch(Exception e){
+ } catch (Exception e) {
batchHelper.fail(e);
batchMutations.clear();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/HBaseLookupBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/HBaseLookupBolt.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/HBaseLookupBolt.java
index d20aab3..538d896 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/HBaseLookupBolt.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/HBaseLookupBolt.java
@@ -1,35 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hbase.bolt;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.tuple.Values;
-import org.apache.storm.utils.TupleUtils;
+package org.apache.storm.hbase.bolt;
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.collect.Lists;
-
import java.util.Map;
import java.util.concurrent.TimeUnit;
-
import org.apache.commons.lang.Validate;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
@@ -38,6 +26,10 @@ import org.apache.storm.hbase.bolt.mapper.HBaseProjectionCriteria;
import org.apache.storm.hbase.bolt.mapper.HBaseValueMapper;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
+import org.apache.storm.utils.TupleUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,82 +37,81 @@ import org.slf4j.LoggerFactory;
* Basic bolt for querying from HBase.
*
* Note: Each HBaseBolt defined in a topology is tied to a specific table.
- *
*/
public class HBaseLookupBolt extends AbstractHBaseBolt {
- private static final Logger LOG = LoggerFactory.getLogger(HBaseLookupBolt.class);
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseLookupBolt.class);
- private HBaseValueMapper rowToTupleMapper;
- private HBaseProjectionCriteria projectionCriteria;
- private transient LoadingCache<byte[], Result> cache;
- private transient boolean cacheEnabled;
+ private HBaseValueMapper rowToTupleMapper;
+ private HBaseProjectionCriteria projectionCriteria;
+ private transient LoadingCache<byte[], Result> cache;
+ private transient boolean cacheEnabled;
- public HBaseLookupBolt(String tableName, HBaseMapper mapper, HBaseValueMapper rowToTupleMapper) {
- super(tableName, mapper);
- Validate.notNull(rowToTupleMapper, "rowToTupleMapper can not be null");
- this.rowToTupleMapper = rowToTupleMapper;
- }
+ public HBaseLookupBolt(String tableName, HBaseMapper mapper, HBaseValueMapper rowToTupleMapper) {
+ super(tableName, mapper);
+ Validate.notNull(rowToTupleMapper, "rowToTupleMapper can not be null");
+ this.rowToTupleMapper = rowToTupleMapper;
+ }
- public HBaseLookupBolt withConfigKey(String configKey) {
- this.configKey = configKey;
- return this;
- }
+ public HBaseLookupBolt withConfigKey(String configKey) {
+ this.configKey = configKey;
+ return this;
+ }
- public HBaseLookupBolt withProjectionCriteria(HBaseProjectionCriteria projectionCriteria) {
- this.projectionCriteria = projectionCriteria;
- return this;
- }
+ public HBaseLookupBolt withProjectionCriteria(HBaseProjectionCriteria projectionCriteria) {
+ this.projectionCriteria = projectionCriteria;
+ return this;
+ }
- @Override
- public void prepare(Map<String, Object> config, TopologyContext topologyContext, OutputCollector collector) {
- super.prepare(config, topologyContext, collector);
- cacheEnabled = Boolean.parseBoolean(config.getOrDefault("hbase.cache.enable", "false").toString());
- int cacheTTL = Integer.parseInt(config.getOrDefault("hbase.cache.ttl.seconds", "300").toString());
- int maxCacheSize = Integer.parseInt(config.getOrDefault("hbase.cache.size", "1000").toString());
- if (cacheEnabled) {
- cache = Caffeine.newBuilder().maximumSize(maxCacheSize).expireAfterWrite(cacheTTL, TimeUnit.SECONDS)
- .build(new CacheLoader<byte[], Result>() {
+ @Override
+ public void prepare(Map<String, Object> config, TopologyContext topologyContext, OutputCollector collector) {
+ super.prepare(config, topologyContext, collector);
+ cacheEnabled = Boolean.parseBoolean(config.getOrDefault("hbase.cache.enable", "false").toString());
+ int cacheTTL = Integer.parseInt(config.getOrDefault("hbase.cache.ttl.seconds", "300").toString());
+ int maxCacheSize = Integer.parseInt(config.getOrDefault("hbase.cache.size", "1000").toString());
+ if (cacheEnabled) {
+ cache = Caffeine.newBuilder().maximumSize(maxCacheSize).expireAfterWrite(cacheTTL, TimeUnit.SECONDS)
+ .build(new CacheLoader<byte[], Result>() {
- @Override
- public Result load(byte[] rowKey) throws Exception {
- Get get = hBaseClient.constructGetRequests(rowKey, projectionCriteria);
- if (LOG.isDebugEnabled()) {
+ @Override
+ public Result load(byte[] rowKey) throws Exception {
+ Get get = hBaseClient.constructGetRequests(rowKey, projectionCriteria);
+ if (LOG.isDebugEnabled()) {
LOG.debug("Cache miss for key:" + new String(rowKey));
- }
- return hBaseClient.batchGet(Lists.newArrayList(get))[0];
- }
+ }
+ return hBaseClient.batchGet(Lists.newArrayList(get))[0];
+ }
- });
- }
- }
+ });
+ }
+ }
- @Override
- public void execute(Tuple tuple) {
- if (TupleUtils.isTick(tuple)) {
- collector.ack(tuple);
- return;
- }
- byte[] rowKey = this.mapper.rowKey(tuple);
- Result result = null;
- try {
- if (cacheEnabled) {
- result = cache.get(rowKey);
- } else {
- Get get = hBaseClient.constructGetRequests(rowKey, projectionCriteria);
- result = hBaseClient.batchGet(Lists.newArrayList(get))[0];
- }
- for (Values values : rowToTupleMapper.toValues(tuple, result)) {
- this.collector.emit(tuple, values);
- }
- this.collector.ack(tuple);
- } catch (Exception e) {
- this.collector.reportError(e);
- this.collector.fail(tuple);
- }
- }
+ @Override
+ public void execute(Tuple tuple) {
+ if (TupleUtils.isTick(tuple)) {
+ collector.ack(tuple);
+ return;
+ }
+ byte[] rowKey = this.mapper.rowKey(tuple);
+ Result result = null;
+ try {
+ if (cacheEnabled) {
+ result = cache.get(rowKey);
+ } else {
+ Get get = hBaseClient.constructGetRequests(rowKey, projectionCriteria);
+ result = hBaseClient.batchGet(Lists.newArrayList(get))[0];
+ }
+ for (Values values : rowToTupleMapper.toValues(tuple, result)) {
+ this.collector.emit(tuple, values);
+ }
+ this.collector.ack(tuple);
+ } catch (Exception e) {
+ this.collector.reportError(e);
+ this.collector.fail(tuple);
+ }
+ }
- @Override
- public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
- rowToTupleMapper.declareOutputFields(outputFieldsDeclarer);
- }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
+ rowToTupleMapper.declareOutputFields(outputFieldsDeclarer);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseMapper.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseMapper.java
index 70016c5..bf3eca6 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseMapper.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseMapper.java
@@ -1,31 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hbase.bolt.mapper;
+package org.apache.storm.hbase.bolt.mapper;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.hbase.common.ColumnList;
import java.io.Serializable;
+import org.apache.storm.hbase.common.ColumnList;
+import org.apache.storm.tuple.Tuple;
/**
- * Maps a <code>org.apache.storm.tuple.Tuple</code> object
- * to a row in an HBase table.
+ * Maps a <code>org.apache.storm.tuple.Tuple</code> object to a row in an HBase table.
*/
public interface HBaseMapper extends Serializable {
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseProjectionCriteria.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseProjectionCriteria.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseProjectionCriteria.java
index 7325c62..18322c6 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseProjectionCriteria.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseProjectionCriteria.java
@@ -1,61 +1,30 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.bolt.mapper;
import com.google.common.collect.Lists;
-
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
/**
- * Allows the user to specify the projection criteria.
- * If only columnFamily is specified all columns from that family will be returned.
- * If a column is specified only that column from that family will be returned.
-
+ * Allows the user to specify the projection criteria. If only columnFamily is specified all columns from that family will be returned. If a
+ * column is specified only that column from that family will be returned.
*/
public class HBaseProjectionCriteria implements Serializable {
private List<byte[]> columnFamilies;
private List<ColumnMetaData> columns;
- public static class ColumnMetaData implements Serializable {
- private byte[] columnFamily;
- private byte[] qualifier;
-
- public ColumnMetaData(String columnFamily, String qualifier) {
- this.columnFamily = columnFamily.getBytes();
- this.qualifier = qualifier.getBytes();
- }
-
- public ColumnMetaData(byte[] columnFamily, byte[] qualifier) {
- this.columnFamily = Arrays.copyOf(columnFamily, columnFamily.length);
- this.qualifier = Arrays.copyOf(qualifier, qualifier.length);
- }
-
- public byte[] getColumnFamily() {
- return columnFamily;
- }
-
- public byte[] getQualifier() {
- return qualifier;
- }
- }
-
public HBaseProjectionCriteria() {
columnFamilies = Lists.newArrayList();
columns = Lists.newArrayList();
@@ -63,6 +32,7 @@ public class HBaseProjectionCriteria implements Serializable {
/**
* all columns from this family will be included as result of HBase lookup.
+ *
* @param columnFamily
* @return
*/
@@ -73,6 +43,7 @@ public class HBaseProjectionCriteria implements Serializable {
/**
* all columns from this family will be included as result of HBase lookup.
+ *
* @param columnFamily
* @return
*/
@@ -83,6 +54,7 @@ public class HBaseProjectionCriteria implements Serializable {
/**
* Only this column from the the columnFamily will be included as result of HBase lookup.
+ *
* @param column
* @return
*/
@@ -98,4 +70,27 @@ public class HBaseProjectionCriteria implements Serializable {
public List<byte[]> getColumnFamilies() {
return columnFamilies;
}
+
+ public static class ColumnMetaData implements Serializable {
+ private byte[] columnFamily;
+ private byte[] qualifier;
+
+ public ColumnMetaData(String columnFamily, String qualifier) {
+ this.columnFamily = columnFamily.getBytes();
+ this.qualifier = qualifier.getBytes();
+ }
+
+ public ColumnMetaData(byte[] columnFamily, byte[] qualifier) {
+ this.columnFamily = Arrays.copyOf(columnFamily, columnFamily.length);
+ this.qualifier = Arrays.copyOf(qualifier, qualifier.length);
+ }
+
+ public byte[] getColumnFamily() {
+ return columnFamily;
+ }
+
+ public byte[] getQualifier() {
+ return qualifier;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseValueMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseValueMapper.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseValueMapper.java
index 38e879f..004cf03 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseValueMapper.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/HBaseValueMapper.java
@@ -1,42 +1,37 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.bolt.mapper;
+import java.io.Serializable;
+import java.util.List;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.ITuple;
import org.apache.storm.tuple.Values;
-import org.apache.hadoop.hbase.client.Result;
-
-import java.io.Serializable;
-import java.util.List;
public interface HBaseValueMapper extends Serializable {
/**
- *
- * @param input tuple.
+ * @param input tuple.
* @param result HBase lookup result instance.
* @return list of values that should be emitted by the lookup bolt.
+ *
* @throws Exception
*/
public List<Values> toValues(ITuple input, Result result) throws Exception;
/**
* declares the output fields for the lookup bolt.
+ *
* @param declarer
*/
void declareOutputFields(OutputFieldsDeclarer declarer);
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/SimpleHBaseMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/SimpleHBaseMapper.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/SimpleHBaseMapper.java
index 8747405..f37995c 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/SimpleHBaseMapper.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/bolt/mapper/SimpleHBaseMapper.java
@@ -1,69 +1,66 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.bolt.mapper;
+import org.apache.storm.hbase.common.ColumnList;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
-import org.apache.storm.hbase.common.ColumnList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.storm.hbase.common.Utils.*;
+
+import static org.apache.storm.hbase.common.Utils.toBytes;
+import static org.apache.storm.hbase.common.Utils.toLong;
/**
*
*/
public class SimpleHBaseMapper implements HBaseMapper {
private static final Logger LOG = LoggerFactory.getLogger(SimpleHBaseMapper.class);
-
+
private String rowKeyField;
-// private String timestampField;
+ // private String timestampField;
private byte[] columnFamily;
private Fields columnFields;
private Fields counterFields;
- public SimpleHBaseMapper(){
+ public SimpleHBaseMapper() {
}
- public SimpleHBaseMapper withRowKeyField(String rowKeyField){
+ public SimpleHBaseMapper withRowKeyField(String rowKeyField) {
this.rowKeyField = rowKeyField;
return this;
}
- public SimpleHBaseMapper withColumnFields(Fields columnFields){
+ public SimpleHBaseMapper withColumnFields(Fields columnFields) {
this.columnFields = columnFields;
return this;
}
- public SimpleHBaseMapper withCounterFields(Fields counterFields){
+ public SimpleHBaseMapper withCounterFields(Fields counterFields) {
this.counterFields = counterFields;
return this;
}
- public SimpleHBaseMapper withColumnFamily(String columnFamily){
+ public SimpleHBaseMapper withColumnFamily(String columnFamily) {
this.columnFamily = columnFamily.getBytes();
return this;
}
-// public SimpleTridentHBaseMapper withTimestampField(String timestampField){
-// this.timestampField = timestampField;
-// return this;
-// }
+ // public SimpleTridentHBaseMapper withTimestampField(String timestampField){
+ // this.timestampField = timestampField;
+ // return this;
+ // }
@Override
public byte[] rowKey(Tuple tuple) {
@@ -74,14 +71,14 @@ public class SimpleHBaseMapper implements HBaseMapper {
@Override
public ColumnList columns(Tuple tuple) {
ColumnList cols = new ColumnList();
- if(this.columnFields != null){
+ if (this.columnFields != null) {
// TODO timestamps
- for(String field : this.columnFields){
+ for (String field : this.columnFields) {
cols.addColumn(this.columnFamily, field.getBytes(), toBytes(tuple.getValueByField(field)));
}
}
- if(this.counterFields != null){
- for(String field : this.counterFields){
+ if (this.counterFields != null) {
+ for (String field : this.counterFields) {
cols.addCounter(this.columnFamily, field.getBytes(), toLong(tuple.getValueByField(field)));
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/ColumnList.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/ColumnList.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/ColumnList.java
index 0abe1ad..045b2e5 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/ColumnList.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/ColumnList.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.common;
import java.util.ArrayList;
@@ -28,85 +23,33 @@ import java.util.List;
* Standard columns have <i>column family</i> (required), <i>qualifier</i> (optional),
* <i>timestamp</i> (optional), and a <i>value</i> (optional) values.
*
- * Counter columns have <i>column family</i> (required), <i>qualifier</i> (optional),
- * and an <i>increment</i> (optional, but recommended) values.
- *
- * Inserts/Updates can be added via the <code>addColumn()</code> and <code>addCounter()</code>
- * methods.
+ * Counter columns have <i>column family</i> (required), <i>qualifier</i> (optional), and an <i>increment</i> (optional, but recommended)
+ * values.
*
+ * Inserts/Updates can be added via the <code>addColumn()</code> and <code>addCounter()</code> methods.
*/
public class ColumnList {
- public static abstract class AbstractColumn {
- byte[] family, qualifier;
-
- AbstractColumn(byte[] family, byte[] qualifier){
- this.family = family;
- this.qualifier = qualifier;
- }
-
- public byte[] getFamily() {
- return family;
- }
-
- public byte[] getQualifier() {
- return qualifier;
- }
-
- }
-
- public static class Column extends AbstractColumn {
- byte[] value;
- long ts = -1;
- Column(byte[] family, byte[] qualifier, long ts, byte[] value){
- super(family, qualifier);
- this.value = value;
- this.ts = ts;
- }
-
- public byte[] getValue() {
- return value;
- }
-
- public long getTs() {
- return ts;
- }
- }
-
- public static class Counter extends AbstractColumn {
- long incr = 0;
- Counter(byte[] family, byte[] qualifier, long incr){
- super(family, qualifier);
- this.incr = incr;
- }
-
- public long getIncrement() {
- return incr;
- }
- }
-
-
private ArrayList<Column> columns;
private ArrayList<Column> columnsToDelete;
private ArrayList<Counter> counters;
-
- private ArrayList<Column> columns(){
- if(this.columns == null){
+ private ArrayList<Column> columns() {
+ if (this.columns == null) {
this.columns = new ArrayList<Column>();
}
return this.columns;
}
- private ArrayList<Column> columnsToDelete(){
- if(this.columnsToDelete == null){
+ private ArrayList<Column> columnsToDelete() {
+ if (this.columnsToDelete == null) {
this.columnsToDelete = new ArrayList<Column>();
}
return this.columnsToDelete;
}
- private ArrayList<Counter> counters(){
- if(this.counters == null){
+ private ArrayList<Counter> counters() {
+ if (this.counters == null) {
this.counters = new ArrayList<Counter>();
}
return this.counters;
@@ -121,30 +64,31 @@ public class ColumnList {
* @param value
* @return
*/
- public ColumnList addColumn(byte[] family, byte[] qualifier, long ts, byte[] value){
+ public ColumnList addColumn(byte[] family, byte[] qualifier, long ts, byte[] value) {
columns().add(new Column(family, qualifier, ts, value));
return this;
}
/**
* Add a standard HBase column
+ *
* @param family
* @param qualifier
* @param value
* @return
*/
- public ColumnList addColumn(byte[] family, byte[] qualifier, byte[] value){
+ public ColumnList addColumn(byte[] family, byte[] qualifier, byte[] value) {
columns().add(new Column(family, qualifier, -1, value));
return this;
}
/**
- * Add a standard HBase column given an instance of a class that implements
- * the <code>IColumn</code> interface.
+ * Add a standard HBase column given an instance of a class that implements the <code>IColumn</code> interface.
+ *
* @param column
* @return
*/
- public ColumnList addColumn(IColumn column){
+ public ColumnList addColumn(IColumn column) {
return this.addColumn(column.family(), column.qualifier(), column.timestamp(), column.value());
}
@@ -156,7 +100,7 @@ public class ColumnList {
* @param incr
* @return
*/
- public ColumnList addCounter(byte[] family, byte[] qualifier, long incr){
+ public ColumnList addCounter(byte[] family, byte[] qualifier, long incr) {
counters().add(new Counter(family, qualifier, incr));
return this;
}
@@ -164,10 +108,11 @@ public class ColumnList {
/**
* Add an HBase counter column given an instance of a class that implements the
* <code>ICounter</code> interface.
+ *
* @param counter
* @return
*/
- public ColumnList addCounter(ICounter counter){
+ public ColumnList addCounter(ICounter counter) {
return this.addCounter(counter.family(), counter.qualifier(), counter.increment());
}
@@ -188,7 +133,7 @@ public class ColumnList {
*
* @return
*/
- public boolean hasColumns(){
+ public boolean hasColumns() {
return this.columns != null;
}
@@ -197,7 +142,7 @@ public class ColumnList {
*
* @return
*/
- public boolean hasColumnsToDelete(){
+ public boolean hasColumnsToDelete() {
return this.columnsToDelete != null;
}
@@ -206,7 +151,7 @@ public class ColumnList {
*
* @return
*/
- public boolean hasCounters(){
+ public boolean hasCounters() {
return this.counters != null;
}
@@ -215,7 +160,7 @@ public class ColumnList {
*
* @return
*/
- public List<Column> getColumns(){
+ public List<Column> getColumns() {
return this.columns;
}
@@ -230,10 +175,61 @@ public class ColumnList {
/**
* Get the list of counter definitions.
+ *
* @return
*/
- public List<Counter> getCounters(){
+ public List<Counter> getCounters() {
return this.counters;
}
+ public static abstract class AbstractColumn {
+ byte[] family, qualifier;
+
+ AbstractColumn(byte[] family, byte[] qualifier) {
+ this.family = family;
+ this.qualifier = qualifier;
+ }
+
+ public byte[] getFamily() {
+ return family;
+ }
+
+ public byte[] getQualifier() {
+ return qualifier;
+ }
+
+ }
+
+ public static class Column extends AbstractColumn {
+ byte[] value;
+ long ts = -1;
+
+ Column(byte[] family, byte[] qualifier, long ts, byte[] value) {
+ super(family, qualifier);
+ this.value = value;
+ this.ts = ts;
+ }
+
+ public byte[] getValue() {
+ return value;
+ }
+
+ public long getTs() {
+ return ts;
+ }
+ }
+
+ public static class Counter extends AbstractColumn {
+ long incr = 0;
+
+ Counter(byte[] family, byte[] qualifier, long incr) {
+ super(family, qualifier);
+ this.incr = incr;
+ }
+
+ public long getIncrement() {
+ return incr;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/HBaseClient.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/HBaseClient.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/HBaseClient.java
index ad5160c..208e622 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/HBaseClient.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/HBaseClient.java
@@ -1,30 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hbase.common;
import com.google.common.collect.Lists;
-
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Delete;
@@ -51,7 +43,7 @@ public class HBaseClient implements Closeable {
private HTable table;
- public HBaseClient(Map<String, Object> map , final Configuration configuration, final String tableName) {
+ public HBaseClient(Map<String, Object> map, final Configuration configuration, final String tableName) {
try {
UserProvider provider = HBaseSecurityUtil.login(map, configuration);
this.table = Utils.getTable(provider, configuration, tableName);
@@ -69,16 +61,16 @@ public class HBaseClient implements Closeable {
for (ColumnList.Column col : cols.getColumns()) {
if (col.getTs() > 0) {
put.add(
- col.getFamily(),
- col.getQualifier(),
- col.getTs(),
- col.getValue()
+ col.getFamily(),
+ col.getQualifier(),
+ col.getTs(),
+ col.getValue()
);
} else {
put.add(
- col.getFamily(),
- col.getQualifier(),
- col.getValue()
+ col.getFamily(),
+ col.getQualifier(),
+ col.getValue()
);
}
}
@@ -90,9 +82,9 @@ public class HBaseClient implements Closeable {
inc.setDurability(durability);
for (ColumnList.Counter cnt : cols.getCounters()) {
inc.addColumn(
- cnt.getFamily(),
- cnt.getQualifier(),
- cnt.getIncrement()
+ cnt.getFamily(),
+ cnt.getQualifier(),
+ cnt.getIncrement()
);
}
mutations.add(inc);
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/IColumn.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/IColumn.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/IColumn.java
index 36f7e96..8f64c2e 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/IColumn.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/IColumn.java
@@ -1,30 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.common;
/**
- * Interface definition for classes that support being written to HBase as
- * a regular column.
- *
+ * Interface definition for classes that support being written to HBase as a regular column.
*/
public interface IColumn {
byte[] family();
+
byte[] qualifier();
+
byte[] value();
+
long timestamp();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/ICounter.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/ICounter.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/ICounter.java
index 43e3f60..ce9d84f 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/ICounter.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/ICounter.java
@@ -1,29 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.common;
/**
- * Interface definition for classes that support being written to HBase as
- * a counter column.
- *
+ * Interface definition for classes that support being written to HBase as a counter column.
*/
public interface ICounter {
byte[] family();
+
byte[] qualifier();
+
long increment();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/Utils.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/Utils.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/Utils.java
index 981d4ff..0dca16e 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/Utils.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/common/Utils.java
@@ -1,22 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.common;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.security.PrivilegedExceptionAction;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.security.UserProvider;
@@ -27,18 +25,14 @@ import org.apache.hadoop.security.token.TokenIdentifier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.security.PrivilegedExceptionAction;
-
public class Utils {
- private static final Logger LOG = LoggerFactory.getLogger(Utils.class);
public static final String TOKEN_KIND_HBASE_AUTH_TOKEN = "HBASE_AUTH_TOKEN";
+ private static final Logger LOG = LoggerFactory.getLogger(Utils.class);
- private Utils(){}
+ private Utils() {}
public static HTable getTable(UserProvider provider, Configuration config, String tableName)
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException {
UserGroupInformation ugi;
if (provider != null) {
ugi = provider.getCurrent().getUGI();
@@ -51,7 +45,7 @@ public class Utils {
boolean foundHBaseAuthToken = false;
for (Token<? extends TokenIdentifier> token : ugi.getTokens()) {
LOG.debug("Token in UGI (delegation token): {} / {}", token.toString(),
- token.decodeIdentifier().getUser());
+ token.decodeIdentifier().getUser());
// token.getKind() = Text, Text is annotated by @Stringable
// which ensures toString() implementation
@@ -66,7 +60,7 @@ public class Utils {
foundHBaseAuthToken = true;
} else {
LOG.warn("Found multiple HBASE_AUTH_TOKEN - will use already found token. " +
- "Please enable DEBUG log level to track delegation tokens.");
+ "Please enable DEBUG log level to track delegation tokens.");
}
}
}
@@ -78,18 +72,19 @@ public class Utils {
}
return ugi.doAs(new PrivilegedExceptionAction<HTable>() {
- @Override public HTable run() throws IOException {
+ @Override
+ public HTable run() throws IOException {
return new HTable(config, tableName);
}
});
}
- public static long toLong(Object obj){
+ public static long toLong(Object obj) {
long l = 0;
- if(obj != null){
- if(obj instanceof Number){
- l = ((Number)obj).longValue();
- } else{
+ if (obj != null) {
+ if (obj instanceof Number) {
+ l = ((Number) obj).longValue();
+ } else {
LOG.warn("Could not coerce {} to Long", obj.getClass().getName());
}
}
@@ -97,24 +92,24 @@ public class Utils {
}
public static byte[] toBytes(Object obj) {
- if(obj == null) {
- return null;
- } else if(obj instanceof String){
- return ((String)obj).getBytes();
- } else if (obj instanceof Integer){
+ if (obj == null) {
+ return null;
+ } else if (obj instanceof String) {
+ return ((String) obj).getBytes();
+ } else if (obj instanceof Integer) {
return Bytes.toBytes((Integer) obj);
- } else if (obj instanceof Long){
- return Bytes.toBytes((Long)obj);
- } else if (obj instanceof Short){
- return Bytes.toBytes((Short)obj);
- } else if (obj instanceof Float){
- return Bytes.toBytes((Float)obj);
- } else if (obj instanceof Double){
- return Bytes.toBytes((Double)obj);
- } else if (obj instanceof Boolean){
- return Bytes.toBytes((Boolean)obj);
- } else if (obj instanceof BigDecimal){
- return Bytes.toBytes((BigDecimal)obj);
+ } else if (obj instanceof Long) {
+ return Bytes.toBytes((Long) obj);
+ } else if (obj instanceof Short) {
+ return Bytes.toBytes((Short) obj);
+ } else if (obj instanceof Float) {
+ return Bytes.toBytes((Float) obj);
+ } else if (obj instanceof Double) {
+ return Bytes.toBytes((Double) obj);
+ } else if (obj instanceof Boolean) {
+ return Bytes.toBytes((Boolean) obj);
+ } else if (obj instanceof BigDecimal) {
+ return Bytes.toBytes((BigDecimal) obj);
} else {
LOG.error("Can't convert class to byte array: " + obj.getClass().getName());
return new byte[0];
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueState.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueState.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueState.java
index dcbaf65..0da456c 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueState.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueState.java
@@ -20,7 +20,6 @@ package org.apache.storm.hbase.state;
import com.google.common.collect.Maps;
import com.google.common.primitives.UnsignedBytes;
-
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -31,7 +30,6 @@ import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
-
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
@@ -51,14 +49,11 @@ import org.slf4j.LoggerFactory;
* A Hbase based implementation that persists the state in HBase.
*/
public class HBaseKeyValueState<K, V> implements KeyValueState<K, V> {
- private static final Logger LOG = LoggerFactory.getLogger(HBaseKeyValueState.class);
-
- public static byte[] STATE_QUALIFIER = "s".getBytes();
public static final int ITERATOR_CHUNK_SIZE = 1000;
-
public static final NavigableMap<byte[], byte[]> EMPTY_PENDING_COMMIT_MAP = Maps.unmodifiableNavigableMap(
- new TreeMap<byte[], byte[]>(UnsignedBytes.lexicographicalComparator()));
-
+ new TreeMap<byte[], byte[]>(UnsignedBytes.lexicographicalComparator()));
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseKeyValueState.class);
+ public static byte[] STATE_QUALIFIER = "s".getBytes();
private static byte[] COMMIT_TXID_KEY = "commit".getBytes();
private static byte[] PREPARE_TXID_KEY = "prepare".getBytes();
@@ -79,22 +74,22 @@ public class HBaseKeyValueState<K, V> implements KeyValueState<K, V> {
/**
* Constructor.
*
- * @param hbaseClient HBaseClient instance
+ * @param hbaseClient HBaseClient instance
* @param columnFamily column family to store State
- * @param namespace namespace
+ * @param namespace namespace
*/
public HBaseKeyValueState(HBaseClient hbaseClient, String columnFamily, String namespace) {
this(hbaseClient, columnFamily, namespace, new DefaultStateSerializer<K>(),
- new DefaultStateSerializer<V>());
+ new DefaultStateSerializer<V>());
}
/**
* Constructor.
*
- * @param hbaseClient HBaseClient instance
- * @param columnFamily column family to store State
- * @param namespace namespace
- * @param keySerializer key serializer
+ * @param hbaseClient HBaseClient instance
+ * @param columnFamily column family to store State
+ * @param namespace namespace
+ * @param keySerializer key serializer
* @param valueSerializer value serializer
*/
public HBaseKeyValueState(HBaseClient hbaseClient, String columnFamily, String namespace,
@@ -173,7 +168,7 @@ public class HBaseKeyValueState<K, V> implements KeyValueState<K, V> {
} else {
HBaseProjectionCriteria criteria = new HBaseProjectionCriteria();
HBaseProjectionCriteria.ColumnMetaData column = new HBaseProjectionCriteria.ColumnMetaData(columnFamily,
- STATE_QUALIFIER);
+ STATE_QUALIFIER);
criteria.addColumn(column);
Get get = hbaseClient.constructGetRequests(getRowKeyForStateKey(columnKey), criteria);
try {
@@ -210,7 +205,8 @@ public class HBaseKeyValueState<K, V> implements KeyValueState<K, V> {
@Override
public Iterator<Map.Entry<K, V>> iterator() {
return new HBaseKeyValueStateIterator<>(namespace, columnFamily, hbaseClient, pendingPrepare.entrySet().iterator(),
- pendingCommit.entrySet().iterator(), ITERATOR_CHUNK_SIZE, encoder.getKeySerializer(), encoder.getValueSerializer());
+ pendingCommit.entrySet().iterator(), ITERATOR_CHUNK_SIZE, encoder.getKeySerializer(),
+ encoder.getValueSerializer());
}
@Override
@@ -316,7 +312,7 @@ public class HBaseKeyValueState<K, V> implements KeyValueState<K, V> {
if (committedTxid != null) {
if (txid <= committedTxid) {
throw new RuntimeException("Invalid txid '" + txid + "' for prepare. Txid '" + committedTxid
- + "' is already committed");
+ + "' is already committed");
}
}
}
@@ -375,7 +371,7 @@ public class HBaseKeyValueState<K, V> implements KeyValueState<K, V> {
mutations.add(new Delete(getRowKeyForStateKey(rowKey)));
} else {
List<Mutation> mutationsForRow = prepareMutateRow(getRowKeyForStateKey(rowKey), columnFamily,
- Collections.singletonMap(STATE_QUALIFIER, value));
+ Collections.singletonMap(STATE_QUALIFIER, value));
mutations.addAll(mutationsForRow);
}
}
@@ -402,7 +398,7 @@ public class HBaseKeyValueState<K, V> implements KeyValueState<K, V> {
}
private void mutateRow(byte[] rowKey, byte[] columnFamily, Map<byte[], byte[]> map)
- throws Exception {
+ throws Exception {
mutateRow(rowKey, columnFamily, map, Durability.USE_DEFAULT);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueStateIterator.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueStateIterator.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueStateIterator.java
index 9b69aec..dbbd6d0 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueStateIterator.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueStateIterator.java
@@ -1,67 +1,57 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hbase.state;
-import static org.apache.storm.hbase.state.HBaseKeyValueState.STATE_QUALIFIER;
-
import com.google.common.primitives.UnsignedBytes;
-
import java.util.Arrays;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
-
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.storm.hbase.common.HBaseClient;
import org.apache.storm.state.BaseBinaryStateIterator;
import org.apache.storm.state.DefaultStateEncoder;
import org.apache.storm.state.Serializer;
-
import org.apache.storm.state.StateEncoder;
+import static org.apache.storm.hbase.state.HBaseKeyValueState.STATE_QUALIFIER;
+
/**
* An iterator over {@link HBaseKeyValueState}.
*/
public class HBaseKeyValueStateIterator<K, V> extends BaseBinaryStateIterator<K, V> {
private final byte[] keyNamespace;
- private byte[] cursorKey;
private final byte[] endScanKey;
private final byte[] columnFamily;
private final HBaseClient hbaseClient;
private final int chunkSize;
private final StateEncoder<K, V, byte[], byte[]> encoder;
-
+ private byte[] cursorKey;
private Iterator<Map.Entry<byte[], byte[]>> cachedResultIterator;
/**
* Constructor.
*
- * @param namespace The namespace of State
- * @param columnFamily The column family of state
- * @param hbaseClient The instance of HBaseClient
+ * @param namespace The namespace of State
+ * @param columnFamily The column family of state
+ * @param hbaseClient The instance of HBaseClient
* @param pendingPrepareIterator The iterator of pendingPrepare
- * @param pendingCommitIterator The iterator of pendingCommit
- * @param chunkSize The size of chunk to get entries from HBase
- * @param keySerializer The serializer of key
- * @param valueSerializer The serializer of value
+ * @param pendingCommitIterator The iterator of pendingCommit
+ * @param chunkSize The size of chunk to get entries from HBase
+ * @param keySerializer The serializer of key
+ * @param valueSerializer The serializer of value
*/
public HBaseKeyValueStateIterator(String namespace, byte[] columnFamily, HBaseClient hbaseClient,
Iterator<Map.Entry<byte[], byte[]>> pendingPrepareIterator,
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueStateProvider.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueStateProvider.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueStateProvider.java
index 763c387..89adae3 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueStateProvider.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/state/HBaseKeyValueStateProvider.java
@@ -21,6 +21,9 @@ package org.apache.storm.hbase.state;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.procedure2.util.StringUtils;
@@ -34,10 +37,6 @@ import org.apache.storm.task.TopologyContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
/**
* Provides {@link HBaseKeyValueState}.
*/
@@ -86,12 +85,12 @@ public class HBaseKeyValueStateProvider implements StateProvider {
HBaseClient hbaseClient = new HBaseClient(hbaseConfMap, hbConfig, config.tableName);
return new HBaseKeyValueState(hbaseClient, config.columnFamily, namespace,
- getKeySerializer(stormConf, context, config), getValueSerializer(stormConf, context, config));
+ getKeySerializer(stormConf, context, config), getValueSerializer(stormConf, context, config));
}
private Configuration getHBaseConfigurationInstance(Map<String, Object> conf) {
final Configuration hbConfig = HBaseConfiguration.create();
- for(String key : conf.keySet()) {
+ for (String key : conf.keySet()) {
hbConfig.set(key, String.valueOf(conf.get(key)));
}
return hbConfig;
@@ -99,11 +98,11 @@ public class HBaseKeyValueStateProvider implements StateProvider {
private Map<String, Object> getHBaseConfigMap(Map<String, Object> stormConfMap, String hbaseConfigKey) {
Map<String, Object> conf = (Map<String, Object>) stormConfMap.get(hbaseConfigKey);
- if(conf == null) {
+ if (conf == null) {
throw new IllegalArgumentException("HBase configuration not found using key '" + hbaseConfigKey + "'");
}
- if(conf.get("hbase.rootdir") == null) {
+ if (conf.get("hbase.rootdir") == null) {
LOG.warn("No 'hbase.rootdir' value found in configuration! Using HBase defaults.");
}
return conf;
@@ -153,14 +152,14 @@ public class HBaseKeyValueStateProvider implements StateProvider {
@Override
public String toString() {
return "StateConfig{" +
- "keyClass='" + keyClass + '\'' +
- ", valueClass='" + valueClass + '\'' +
- ", keySerializerClass='" + keySerializerClass + '\'' +
- ", valueSerializerClass='" + valueSerializerClass + '\'' +
- ", hbaseConfigKey='" + hbaseConfigKey + '\'' +
- ", tableName='" + tableName + '\'' +
- ", columnFamily='" + columnFamily + '\'' +
- '}';
+ "keyClass='" + keyClass + '\'' +
+ ", valueClass='" + valueClass + '\'' +
+ ", keySerializerClass='" + keySerializerClass + '\'' +
+ ", valueSerializerClass='" + valueSerializerClass + '\'' +
+ ", hbaseConfigKey='" + hbaseConfigKey + '\'' +
+ ", tableName='" + tableName + '\'' +
+ ", columnFamily='" + columnFamily + '\'' +
+ '}';
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/SimpleTridentHBaseMapMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/SimpleTridentHBaseMapMapper.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/SimpleTridentHBaseMapMapper.java
index 2262a79..86f38d7 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/SimpleTridentHBaseMapMapper.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/SimpleTridentHBaseMapMapper.java
@@ -37,7 +37,7 @@ public class SimpleTridentHBaseMapMapper implements TridentHBaseMapMapper {
bos.write(String.valueOf(key).getBytes());
}
bos.close();
- } catch (IOException e){
+ } catch (IOException e) {
throw new RuntimeException("IOException creating HBase row key.", e);
}
return bos.toByteArray();
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/SimpleTridentHBaseMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/SimpleTridentHBaseMapper.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/SimpleTridentHBaseMapper.java
index eda9a32..f7ec2c7 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/SimpleTridentHBaseMapper.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/SimpleTridentHBaseMapper.java
@@ -1,29 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.trident.mapper;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.hbase.bolt.mapper.HBaseMapper;
import org.apache.storm.hbase.common.ColumnList;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Fields;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.storm.trident.tuple.TridentTuple;
import static org.apache.storm.hbase.common.Utils.toBytes;
import static org.apache.storm.hbase.common.Utils.toLong;
@@ -33,32 +26,32 @@ import static org.apache.storm.hbase.common.Utils.toLong;
*/
public class SimpleTridentHBaseMapper implements TridentHBaseMapper {
private static final Logger LOG = LoggerFactory.getLogger(SimpleTridentHBaseMapper.class);
-
+
private String rowKeyField;
private byte[] columnFamily;
private Fields columnFields;
private Fields counterFields;
- public SimpleTridentHBaseMapper(){
+ public SimpleTridentHBaseMapper() {
}
- public SimpleTridentHBaseMapper withRowKeyField(String rowKeyField){
+ public SimpleTridentHBaseMapper withRowKeyField(String rowKeyField) {
this.rowKeyField = rowKeyField;
return this;
}
- public SimpleTridentHBaseMapper withColumnFields(Fields columnFields){
+ public SimpleTridentHBaseMapper withColumnFields(Fields columnFields) {
this.columnFields = columnFields;
return this;
}
- public SimpleTridentHBaseMapper withCounterFields(Fields counterFields){
+ public SimpleTridentHBaseMapper withCounterFields(Fields counterFields) {
this.counterFields = counterFields;
return this;
}
- public SimpleTridentHBaseMapper withColumnFamily(String columnFamily){
+ public SimpleTridentHBaseMapper withColumnFamily(String columnFamily) {
this.columnFamily = columnFamily.getBytes();
return this;
}
@@ -73,14 +66,14 @@ public class SimpleTridentHBaseMapper implements TridentHBaseMapper {
@Override
public ColumnList columns(TridentTuple tuple) {
ColumnList cols = new ColumnList();
- if(this.columnFields != null){
+ if (this.columnFields != null) {
// TODO timestamps
- for(String field : this.columnFields){
+ for (String field : this.columnFields) {
cols.addColumn(this.columnFamily, field.getBytes(), toBytes(tuple.getValueByField(field)));
}
}
- if(this.counterFields != null){
- for(String field : this.counterFields){
+ if (this.counterFields != null) {
+ for (String field : this.counterFields) {
cols.addCounter(this.columnFamily, field.getBytes(), toLong(tuple.getValueByField(field)));
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/TridentHBaseMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/TridentHBaseMapper.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/TridentHBaseMapper.java
index bb95497..dfdc2b2 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/TridentHBaseMapper.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/mapper/TridentHBaseMapper.java
@@ -19,31 +19,29 @@
package org.apache.storm.hbase.trident.mapper;
-import org.apache.storm.tuple.Tuple;
+import java.io.Serializable;
import org.apache.storm.hbase.common.ColumnList;
import org.apache.storm.trident.tuple.TridentTuple;
-import java.io.Serializable;
/**
- * Maps a <code>org.apache.storm.trident.tuple.TridentTuple</code> object
- * to a row in an HBase table.
+ * Maps a <code>org.apache.storm.trident.tuple.TridentTuple</code> object to a row in an HBase table.
*/
public interface TridentHBaseMapper extends Serializable {
- /**
- * Given a tuple, return the HBase rowkey.
- *
- * @param tuple
- * @return
- */
- byte[] rowKey(TridentTuple tuple);
+ /**
+ * Given a tuple, return the HBase rowkey.
+ *
+ * @param tuple
+ * @return
+ */
+ byte[] rowKey(TridentTuple tuple);
- /**
- * Given a tuple, return a list of HBase columns to insert.
- *
- * @param tuple
- * @return
- */
- ColumnList columns(TridentTuple tuple);
+ /**
+ * Given a tuple, return a list of HBase columns to insert.
+ *
+ * @param tuple
+ * @return
+ */
+ ColumnList columns(TridentTuple tuple);
}
[23/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-eventhubs
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/EventCount.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/EventCount.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/EventCount.java
index 4709b5e..8d00397 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/EventCount.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/EventCount.java
@@ -15,11 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.samples;
import java.io.FileReader;
import java.util.Properties;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.eventhubs.samples.bolt.GlobalCountBolt;
@@ -30,119 +30,115 @@ import org.apache.storm.generated.StormTopology;
import org.apache.storm.topology.TopologyBuilder;
/**
- * The basic scenario topology that uses EventHubSpout with PartialCountBolt
- * and GlobalCountBolt.
- * To submit this topology:
- * storm jar {jarfile} {classname} {topologyname} {spoutconffile}
+ * The basic scenario topology that uses EventHubSpout with PartialCountBolt and GlobalCountBolt. To submit this topology: storm jar
+ * {jarfile} {classname} {topologyname} {spoutconffile}
*/
public class EventCount {
- protected EventHubSpoutConfig spoutConfig;
- protected int numWorkers;
-
- public EventCount() {
- }
-
- protected void readEHConfig(String[] args) throws Exception {
- Properties properties = new Properties();
- if(args.length > 1) {
- properties.load(new FileReader(args[1]));
+ protected EventHubSpoutConfig spoutConfig;
+ protected int numWorkers;
+
+ public EventCount() {
}
- else {
- properties.load(EventCount.class.getClassLoader().getResourceAsStream(
- "Config.properties"));
+
+ public static void main(String[] args) throws Exception {
+ EventCount scenario = new EventCount();
+ scenario.runScenario(args);
}
- String username = properties.getProperty("eventhubspout.username");
- String password = properties.getProperty("eventhubspout.password");
- String namespaceName = properties.getProperty("eventhubspout.namespace");
- String entityPath = properties.getProperty("eventhubspout.entitypath");
- String targetFqnAddress = properties.getProperty("eventhubspout.targetfqnaddress");
- String zkEndpointAddress = properties.getProperty("zookeeper.connectionstring");
- int partitionCount = Integer.parseInt(properties.getProperty("eventhubspout.partitions.count"));
- int checkpointIntervalInSeconds = Integer.parseInt(properties.getProperty("eventhubspout.checkpoint.interval"));
- int receiverCredits = Integer.parseInt(properties.getProperty("eventhub.receiver.credits"));
- String maxPendingMsgsPerPartitionStr = properties.getProperty("eventhubspout.max.pending.messages.per.partition");
- if(maxPendingMsgsPerPartitionStr == null) {
- maxPendingMsgsPerPartitionStr = "1024";
+ protected void readEHConfig(String[] args) throws Exception {
+ Properties properties = new Properties();
+ if (args.length > 1) {
+ properties.load(new FileReader(args[1]));
+ } else {
+ properties.load(EventCount.class.getClassLoader().getResourceAsStream(
+ "Config.properties"));
+ }
+
+ String username = properties.getProperty("eventhubspout.username");
+ String password = properties.getProperty("eventhubspout.password");
+ String namespaceName = properties.getProperty("eventhubspout.namespace");
+ String entityPath = properties.getProperty("eventhubspout.entitypath");
+ String targetFqnAddress = properties.getProperty("eventhubspout.targetfqnaddress");
+ String zkEndpointAddress = properties.getProperty("zookeeper.connectionstring");
+ int partitionCount = Integer.parseInt(properties.getProperty("eventhubspout.partitions.count"));
+ int checkpointIntervalInSeconds = Integer.parseInt(properties.getProperty("eventhubspout.checkpoint.interval"));
+ int receiverCredits = Integer.parseInt(properties.getProperty("eventhub.receiver.credits"));
+ String maxPendingMsgsPerPartitionStr = properties.getProperty("eventhubspout.max.pending.messages.per.partition");
+ if (maxPendingMsgsPerPartitionStr == null) {
+ maxPendingMsgsPerPartitionStr = "1024";
+ }
+ int maxPendingMsgsPerPartition = Integer.parseInt(maxPendingMsgsPerPartitionStr);
+ String enqueueTimeDiffStr = properties.getProperty("eventhub.receiver.filter.timediff");
+ if (enqueueTimeDiffStr == null) {
+ enqueueTimeDiffStr = "0";
+ }
+ int enqueueTimeDiff = Integer.parseInt(enqueueTimeDiffStr);
+ long enqueueTimeFilter = 0;
+ if (enqueueTimeDiff != 0) {
+ enqueueTimeFilter = System.currentTimeMillis() - enqueueTimeDiff * 1000;
+ }
+ String consumerGroupName = properties.getProperty("eventhubspout.consumer.group.name");
+
+ System.out.println("Eventhub spout config: ");
+ System.out.println(" partition count: " + partitionCount);
+ System.out.println(" checkpoint interval: " + checkpointIntervalInSeconds);
+ System.out.println(" receiver credits: " + receiverCredits);
+ spoutConfig = new EventHubSpoutConfig(username, password,
+ namespaceName, entityPath, partitionCount, zkEndpointAddress,
+ checkpointIntervalInSeconds, receiverCredits, maxPendingMsgsPerPartition,
+ enqueueTimeFilter);
+
+ if (targetFqnAddress != null) {
+ spoutConfig.setTargetAddress(targetFqnAddress);
+ }
+ spoutConfig.setConsumerGroupName(consumerGroupName);
+
+ //set the number of workers to be the same as partition number.
+ //the idea is to have a spout and a partial count bolt co-exist in one
+ //worker to avoid shuffling messages across workers in storm cluster.
+ numWorkers = spoutConfig.getPartitionCount();
+
+ if (args.length > 0) {
+ //set topology name so that sample Trident topology can use it as stream name.
+ spoutConfig.setTopologyName(args[0]);
+ }
}
- int maxPendingMsgsPerPartition = Integer.parseInt(maxPendingMsgsPerPartitionStr);
- String enqueueTimeDiffStr = properties.getProperty("eventhub.receiver.filter.timediff");
- if(enqueueTimeDiffStr == null) {
- enqueueTimeDiffStr = "0";
+
+ protected EventHubSpout createEventHubSpout() {
+ EventHubSpout eventHubSpout = new EventHubSpout(spoutConfig);
+ return eventHubSpout;
}
- int enqueueTimeDiff = Integer.parseInt(enqueueTimeDiffStr);
- long enqueueTimeFilter = 0;
- if(enqueueTimeDiff != 0) {
- enqueueTimeFilter = System.currentTimeMillis() - enqueueTimeDiff*1000;
+
+ protected StormTopology buildTopology(EventHubSpout eventHubSpout) {
+ TopologyBuilder topologyBuilder = new TopologyBuilder();
+
+ topologyBuilder.setSpout("EventHubsSpout", eventHubSpout, spoutConfig.getPartitionCount())
+ .setNumTasks(spoutConfig.getPartitionCount());
+ topologyBuilder.setBolt("PartialCountBolt", new PartialCountBolt(), spoutConfig.getPartitionCount())
+ .localOrShuffleGrouping("EventHubsSpout").setNumTasks(spoutConfig.getPartitionCount());
+ topologyBuilder.setBolt("GlobalCountBolt", new GlobalCountBolt(), 1)
+ .globalGrouping("PartialCountBolt").setNumTasks(1);
+ return topologyBuilder.createTopology();
}
- String consumerGroupName = properties.getProperty("eventhubspout.consumer.group.name");
-
- System.out.println("Eventhub spout config: ");
- System.out.println(" partition count: " + partitionCount);
- System.out.println(" checkpoint interval: " + checkpointIntervalInSeconds);
- System.out.println(" receiver credits: " + receiverCredits);
- spoutConfig = new EventHubSpoutConfig(username, password,
- namespaceName, entityPath, partitionCount, zkEndpointAddress,
- checkpointIntervalInSeconds, receiverCredits, maxPendingMsgsPerPartition,
- enqueueTimeFilter);
-
- if(targetFqnAddress != null)
- {
- spoutConfig.setTargetAddress(targetFqnAddress);
+
+ protected void submitTopology(String[] args, StormTopology topology) throws Exception {
+ Config config = new Config();
+ config.setDebug(false);
+ //Enable metrics
+ config.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class, 1);
+
+ String topoName = "test";
+ if (args.length > 0) {
+ topoName = args[0];
+ }
+ config.setNumWorkers(numWorkers);
+ StormSubmitter.submitTopology(topoName, config, topology);
}
- spoutConfig.setConsumerGroupName(consumerGroupName);
-
- //set the number of workers to be the same as partition number.
- //the idea is to have a spout and a partial count bolt co-exist in one
- //worker to avoid shuffling messages across workers in storm cluster.
- numWorkers = spoutConfig.getPartitionCount();
-
- if(args.length > 0) {
- //set topology name so that sample Trident topology can use it as stream name.
- spoutConfig.setTopologyName(args[0]);
+
+ protected void runScenario(String[] args) throws Exception {
+ readEHConfig(args);
+ EventHubSpout eventHubSpout = createEventHubSpout();
+ StormTopology topology = buildTopology(eventHubSpout);
+ submitTopology(args, topology);
}
- }
-
- protected EventHubSpout createEventHubSpout() {
- EventHubSpout eventHubSpout = new EventHubSpout(spoutConfig);
- return eventHubSpout;
- }
-
- protected StormTopology buildTopology(EventHubSpout eventHubSpout) {
- TopologyBuilder topologyBuilder = new TopologyBuilder();
-
- topologyBuilder.setSpout("EventHubsSpout", eventHubSpout, spoutConfig.getPartitionCount())
- .setNumTasks(spoutConfig.getPartitionCount());
- topologyBuilder.setBolt("PartialCountBolt", new PartialCountBolt(), spoutConfig.getPartitionCount())
- .localOrShuffleGrouping("EventHubsSpout").setNumTasks(spoutConfig.getPartitionCount());
- topologyBuilder.setBolt("GlobalCountBolt", new GlobalCountBolt(), 1)
- .globalGrouping("PartialCountBolt").setNumTasks(1);
- return topologyBuilder.createTopology();
- }
-
- protected void submitTopology(String[] args, StormTopology topology) throws Exception {
- Config config = new Config();
- config.setDebug(false);
- //Enable metrics
- config.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class, 1);
-
- String topoName = "test";
- if (args.length > 0) {
- topoName = args[0];
- }
- config.setNumWorkers(numWorkers);
- StormSubmitter.submitTopology(topoName, config, topology);
- }
-
- protected void runScenario(String[] args) throws Exception{
- readEHConfig(args);
- EventHubSpout eventHubSpout = createEventHubSpout();
- StormTopology topology = buildTopology(eventHubSpout);
- submitTopology(args, topology);
- }
-
- public static void main(String[] args) throws Exception {
- EventCount scenario = new EventCount();
- scenario.runScenario(args);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/EventHubLoop.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/EventHubLoop.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/EventHubLoop.java
index 32dc0d5..55d4cf2 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/EventHubLoop.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/EventHubLoop.java
@@ -15,38 +15,38 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
-package org.apache.storm.eventhubs.samples;
-import org.apache.storm.generated.StormTopology;
-import org.apache.storm.topology.TopologyBuilder;
+package org.apache.storm.eventhubs.samples;
import org.apache.storm.eventhubs.bolt.EventHubBolt;
import org.apache.storm.eventhubs.bolt.EventHubBoltConfig;
import org.apache.storm.eventhubs.spout.EventHubSpout;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.topology.TopologyBuilder;
/**
* A sample topology that loops message back to EventHub
*/
public class EventHubLoop extends EventCount {
- @Override
- protected StormTopology buildTopology(EventHubSpout eventHubSpout) {
- TopologyBuilder topologyBuilder = new TopologyBuilder();
+ public static void main(String[] args) throws Exception {
+ EventHubLoop scenario = new EventHubLoop();
+ scenario.runScenario(args);
+ }
+
+ @Override
+ protected StormTopology buildTopology(EventHubSpout eventHubSpout) {
+ TopologyBuilder topologyBuilder = new TopologyBuilder();
+
+ topologyBuilder.setSpout("EventHubsSpout", eventHubSpout, spoutConfig.getPartitionCount())
+ .setNumTasks(spoutConfig.getPartitionCount());
+ EventHubBoltConfig boltConfig = new EventHubBoltConfig(spoutConfig.getConnectionString(),
+ spoutConfig.getEntityPath(), true);
- topologyBuilder.setSpout("EventHubsSpout", eventHubSpout, spoutConfig.getPartitionCount())
- .setNumTasks(spoutConfig.getPartitionCount());
- EventHubBoltConfig boltConfig = new EventHubBoltConfig(spoutConfig.getConnectionString(),
- spoutConfig.getEntityPath(), true);
-
- EventHubBolt eventHubBolt = new EventHubBolt(boltConfig);
- int boltTasks = spoutConfig.getPartitionCount();
- topologyBuilder.setBolt("EventHubsBolt", eventHubBolt, boltTasks)
- .localOrShuffleGrouping("EventHubsSpout").setNumTasks(boltTasks);
- return topologyBuilder.createTopology();
- }
-
- public static void main(String[] args) throws Exception {
- EventHubLoop scenario = new EventHubLoop();
- scenario.runScenario(args);
- }
+ EventHubBolt eventHubBolt = new EventHubBolt(boltConfig);
+ int boltTasks = spoutConfig.getPartitionCount();
+ topologyBuilder.setBolt("EventHubsBolt", eventHubBolt, boltTasks)
+ .localOrShuffleGrouping("EventHubsSpout").setNumTasks(boltTasks);
+ return topologyBuilder.createTopology();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/OpaqueTridentEventCount.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/OpaqueTridentEventCount.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/OpaqueTridentEventCount.java
index a433b8b..abcad5a 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/OpaqueTridentEventCount.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/OpaqueTridentEventCount.java
@@ -15,39 +15,40 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.samples;
+import org.apache.storm.eventhubs.samples.TransactionalTridentEventCount.LoggingFilter;
+import org.apache.storm.eventhubs.spout.EventHubSpout;
+import org.apache.storm.eventhubs.trident.OpaqueTridentEventHubSpout;
+import org.apache.storm.generated.StormTopology;
import org.apache.storm.trident.TridentState;
import org.apache.storm.trident.TridentTopology;
import org.apache.storm.trident.operation.builtin.Count;
import org.apache.storm.trident.operation.builtin.Sum;
import org.apache.storm.trident.testing.MemoryMapState;
-import org.apache.storm.generated.StormTopology;
import org.apache.storm.tuple.Fields;
-import org.apache.storm.eventhubs.samples.TransactionalTridentEventCount.LoggingFilter;
-import org.apache.storm.eventhubs.spout.EventHubSpout;
-import org.apache.storm.eventhubs.trident.OpaqueTridentEventHubSpout;
-
/**
* A simple Trident topology uses OpaqueTridentEventHubSpout
*/
public class OpaqueTridentEventCount extends EventCount {
- @Override
- protected StormTopology buildTopology(EventHubSpout eventHubSpout) {
- TridentTopology topology = new TridentTopology();
-
- OpaqueTridentEventHubSpout spout = new OpaqueTridentEventHubSpout(spoutConfig);
- TridentState state = topology.newStream("stream-" + spoutConfig.getTopologyName(), spout)
- .parallelismHint(spoutConfig.getPartitionCount())
- .aggregate(new Count(), new Fields("partial-count"))
- .persistentAggregate(new MemoryMapState.Factory(), new Fields("partial-count"), new Sum(), new Fields("count"));
- state.newValuesStream().each(new Fields("count"), new LoggingFilter("got count: ", 10000));
- return topology.build();
- }
-
- public static void main(String[] args) throws Exception {
- OpaqueTridentEventCount scenario = new OpaqueTridentEventCount();
- scenario.runScenario(args);
- }
+ public static void main(String[] args) throws Exception {
+ OpaqueTridentEventCount scenario = new OpaqueTridentEventCount();
+ scenario.runScenario(args);
+ }
+
+ @Override
+ protected StormTopology buildTopology(EventHubSpout eventHubSpout) {
+ TridentTopology topology = new TridentTopology();
+
+ OpaqueTridentEventHubSpout spout = new OpaqueTridentEventHubSpout(spoutConfig);
+ TridentState state = topology.newStream("stream-" + spoutConfig.getTopologyName(), spout)
+ .parallelismHint(spoutConfig.getPartitionCount())
+ .aggregate(new Count(), new Fields("partial-count"))
+ .persistentAggregate(new MemoryMapState.Factory(), new Fields("partial-count"), new Sum(),
+ new Fields("count"));
+ state.newValuesStream().each(new Fields("count"), new LoggingFilter("got count: ", 10000));
+ return topology.build();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/TransactionalTridentEventCount.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/TransactionalTridentEventCount.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/TransactionalTridentEventCount.java
index 718c229..646dc9c 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/TransactionalTridentEventCount.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/TransactionalTridentEventCount.java
@@ -15,17 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
-package org.apache.storm.eventhubs.samples;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.storm.generated.StormTopology;
-import org.apache.storm.tuple.Fields;
+package org.apache.storm.eventhubs.samples;
import org.apache.storm.eventhubs.spout.EventHubSpout;
import org.apache.storm.eventhubs.trident.TransactionalTridentEventHubSpout;
-
+import org.apache.storm.generated.StormTopology;
import org.apache.storm.trident.TridentState;
import org.apache.storm.trident.TridentTopology;
import org.apache.storm.trident.operation.BaseFilter;
@@ -33,49 +28,54 @@ import org.apache.storm.trident.operation.builtin.Count;
import org.apache.storm.trident.operation.builtin.Sum;
import org.apache.storm.trident.testing.MemoryMapState;
import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Fields;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A simple Trident topology uses TransactionalTridentEventHubSpout
*/
public class TransactionalTridentEventCount extends EventCount {
- public static class LoggingFilter extends BaseFilter {
- private static final long serialVersionUID = 1L;
- private static final Logger logger = LoggerFactory.getLogger(LoggingFilter.class);
- private final String prefix;
- private final long logIntervalMs;
- private long lastTime;
- public LoggingFilter(String prefix, int logIntervalMs) {
- this.prefix = prefix;
- this.logIntervalMs = logIntervalMs;
- lastTime = System.nanoTime();
+ public static void main(String[] args) throws Exception {
+ TransactionalTridentEventCount scenario = new TransactionalTridentEventCount();
+ scenario.runScenario(args);
}
@Override
- public boolean isKeep(TridentTuple tuple) {
- long now = System.nanoTime();
- if(logIntervalMs < (now - lastTime) / 1000000) {
- logger.info(prefix + tuple.toString());
- lastTime = now;
- }
- return false;
+ protected StormTopology buildTopology(EventHubSpout eventHubSpout) {
+ TridentTopology topology = new TridentTopology();
+
+ TransactionalTridentEventHubSpout spout = new TransactionalTridentEventHubSpout(spoutConfig);
+ TridentState state = topology.newStream("stream-" + spoutConfig.getTopologyName(), spout)
+ .parallelismHint(spoutConfig.getPartitionCount())
+ .aggregate(new Count(), new Fields("partial-count"))
+ .persistentAggregate(new MemoryMapState.Factory(), new Fields("partial-count"), new Sum(),
+ new Fields("count"));
+ state.newValuesStream().each(new Fields("count"), new LoggingFilter("got count: ", 10000));
+ return topology.build();
+ }
+
+ public static class LoggingFilter extends BaseFilter {
+ private static final long serialVersionUID = 1L;
+ private static final Logger logger = LoggerFactory.getLogger(LoggingFilter.class);
+ private final String prefix;
+ private final long logIntervalMs;
+ private long lastTime;
+
+ public LoggingFilter(String prefix, int logIntervalMs) {
+ this.prefix = prefix;
+ this.logIntervalMs = logIntervalMs;
+ lastTime = System.nanoTime();
+ }
+
+ @Override
+ public boolean isKeep(TridentTuple tuple) {
+ long now = System.nanoTime();
+ if (logIntervalMs < (now - lastTime) / 1000000) {
+ logger.info(prefix + tuple.toString());
+ lastTime = now;
+ }
+ return false;
+ }
}
- }
-
- @Override
- protected StormTopology buildTopology(EventHubSpout eventHubSpout) {
- TridentTopology topology = new TridentTopology();
-
- TransactionalTridentEventHubSpout spout = new TransactionalTridentEventHubSpout(spoutConfig);
- TridentState state = topology.newStream("stream-" + spoutConfig.getTopologyName(), spout)
- .parallelismHint(spoutConfig.getPartitionCount())
- .aggregate(new Count(), new Fields("partial-count"))
- .persistentAggregate(new MemoryMapState.Factory(), new Fields("partial-count"), new Sum(), new Fields("count"));
- state.newValuesStream().each(new Fields("count"), new LoggingFilter("got count: ", 10000));
- return topology.build();
- }
-
- public static void main(String[] args) throws Exception {
- TransactionalTridentEventCount scenario = new TransactionalTridentEventCount();
- scenario.runScenario(args);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/bolt/GlobalCountBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/bolt/GlobalCountBolt.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/bolt/GlobalCountBolt.java
index 9d94f5f..cf3fb76 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/bolt/GlobalCountBolt.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/bolt/GlobalCountBolt.java
@@ -15,15 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.samples.bolt;
import java.util.HashMap;
import java.util.Map;
-
-import org.apache.storm.utils.TupleUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.storm.Config;
import org.apache.storm.metric.api.IMetric;
import org.apache.storm.task.TopologyContext;
@@ -31,58 +27,61 @@ import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Tuple;
+import org.apache.storm.utils.TupleUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Globally count number of messages
*/
public class GlobalCountBolt extends BaseBasicBolt {
- private static final long serialVersionUID = 1L;
- private static final Logger logger = LoggerFactory
- .getLogger(GlobalCountBolt.class);
- private long globalCount;
- private long globalCountDiff;
- private long lastMetricsTime;
- private long throughput;
-
- @Override
- public void prepare(Map<String, Object> config, TopologyContext context) {
- globalCount = 0;
- globalCountDiff = 0;
- lastMetricsTime = System.nanoTime();
- context.registerMetric("GlobalMessageCount", new IMetric() {
- @Override
- public Object getValueAndReset() {
- long now = System.nanoTime();
- long millis = (now - lastMetricsTime) / 1000000;
- throughput = globalCountDiff / millis * 1000;
- Map<String, Object> values = new HashMap<>();
- values.put("global_count", globalCount);
- values.put("throughput", throughput);
- lastMetricsTime = now;
- globalCountDiff = 0;
- return values;
- }
- }, (Integer)config.get(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS));
- }
+ private static final long serialVersionUID = 1L;
+ private static final Logger logger = LoggerFactory
+ .getLogger(GlobalCountBolt.class);
+ private long globalCount;
+ private long globalCountDiff;
+ private long lastMetricsTime;
+ private long throughput;
- @Override
- public void execute(Tuple tuple, BasicOutputCollector collector) {
- if (TupleUtils.isTick(tuple)) {
- return;
+ @Override
+ public void prepare(Map<String, Object> config, TopologyContext context) {
+ globalCount = 0;
+ globalCountDiff = 0;
+ lastMetricsTime = System.nanoTime();
+ context.registerMetric("GlobalMessageCount", new IMetric() {
+ @Override
+ public Object getValueAndReset() {
+ long now = System.nanoTime();
+ long millis = (now - lastMetricsTime) / 1000000;
+ throughput = globalCountDiff / millis * 1000;
+ Map<String, Object> values = new HashMap<>();
+ values.put("global_count", globalCount);
+ values.put("throughput", throughput);
+ lastMetricsTime = now;
+ globalCountDiff = 0;
+ return values;
+ }
+ }, (Integer) config.get(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS));
}
- int partial = (Integer)tuple.getValueByField("partial_count");
- globalCount += partial;
- globalCountDiff += partial;
- if((globalCountDiff == partial) && (globalCount != globalCountDiff)) {
- //metrics has just been collected, let's also log it
- logger.info("Current throughput (messages/second): " + throughput);
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ if (TupleUtils.isTick(tuple)) {
+ return;
+ }
+
+ int partial = (Integer) tuple.getValueByField("partial_count");
+ globalCount += partial;
+ globalCountDiff += partial;
+ if ((globalCountDiff == partial) && (globalCount != globalCountDiff)) {
+ //metrics has just been collected, let's also log it
+ logger.info("Current throughput (messages/second): " + throughput);
+ }
}
- }
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
-
- }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/bolt/PartialCountBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/bolt/PartialCountBolt.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/bolt/PartialCountBolt.java
index 3763c69..e51dc08 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/bolt/PartialCountBolt.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/samples/bolt/PartialCountBolt.java
@@ -15,14 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.samples.bolt;
import java.util.Map;
-
-import org.apache.storm.utils.TupleUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -30,39 +26,42 @@ import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
+import org.apache.storm.utils.TupleUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Partially count number of messages from EventHubs
*/
public class PartialCountBolt extends BaseBasicBolt {
- private static final long serialVersionUID = 1L;
- private static final Logger logger = LoggerFactory
- .getLogger(PartialCountBolt.class);
- private static final int PartialCountBatchSize = 1000;
-
- private int partialCount;
-
- @Override
- public void prepare(Map<String, Object> topoConf, TopologyContext context) {
- partialCount = 0;
- }
-
- @Override
- public void execute(Tuple tuple, BasicOutputCollector collector) {
- if (TupleUtils.isTick(tuple)) {
- return;
+ private static final long serialVersionUID = 1L;
+ private static final Logger logger = LoggerFactory
+ .getLogger(PartialCountBolt.class);
+ private static final int PartialCountBatchSize = 1000;
+
+ private int partialCount;
+
+ @Override
+ public void prepare(Map<String, Object> topoConf, TopologyContext context) {
+ partialCount = 0;
}
- partialCount++;
- if(partialCount == PartialCountBatchSize) {
- collector.emit(new Values(PartialCountBatchSize));
- partialCount = 0;
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ if (TupleUtils.isTick(tuple)) {
+ return;
+ }
+
+ partialCount++;
+ if (partialCount == PartialCountBatchSize) {
+ collector.emit(new Values(PartialCountBatchSize));
+ partialCount = 0;
+ }
}
- }
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("partial_count"));
- }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("partial_count"));
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/EventHubReceiverMock.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/EventHubReceiverMock.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/EventHubReceiverMock.java
index 09fb60f..7f6e697 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/EventHubReceiverMock.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/EventHubReceiverMock.java
@@ -15,71 +15,70 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import com.microsoft.azure.eventhubs.EventData;
-
import java.util.Map;
/**
- * A mock receiver that emits fake data with offset starting from given offset
- * and increase by 1 each time.
+ * A mock receiver that emits fake data with offset starting from given offset and increase by 1 each time.
*/
public class EventHubReceiverMock implements IEventHubReceiver {
- private static boolean isPaused = false;
- private final String partitionId;
- private long currentOffset;
- private boolean isOpen;
+ private static boolean isPaused = false;
+ private final String partitionId;
+ private long currentOffset;
+ private boolean isOpen;
+
+ public EventHubReceiverMock(String pid) {
+ partitionId = pid;
+ isPaused = false;
+ }
- public EventHubReceiverMock(String pid) {
- partitionId = pid;
- isPaused = false;
- }
-
- /**
- * Use this method to pause/resume all the receivers.
- * If paused all receiver will return null.
- * @param val
- */
- public static void setPause(boolean val) {
- isPaused = val;
- }
+ /**
+ * Use this method to pause/resume all the receivers. If paused all receiver will return null.
+ *
+ * @param val
+ */
+ public static void setPause(boolean val) {
+ isPaused = val;
+ }
- @Override
- public void open(IEventFilter filter) throws EventHubException {
- currentOffset = filter.getOffset() != null ?
+ @Override
+ public void open(IEventFilter filter) throws EventHubException {
+ currentOffset = filter.getOffset() != null ?
Long.parseLong(filter.getOffset()) :
filter.getTime().toEpochMilli();
- isOpen = true;
- }
-
- @Override
- public void close() {
- isOpen = false;
- }
+ isOpen = true;
+ }
- @Override
- public boolean isOpen() {
- return isOpen;
- }
+ @Override
+ public void close() {
+ isOpen = false;
+ }
- @Override
- public EventDataWrap receive() {
- if(isPaused) {
- return null;
+ @Override
+ public boolean isOpen() {
+ return isOpen;
}
- currentOffset++;
+ @Override
+ public EventDataWrap receive() {
+ if (isPaused) {
+ return null;
+ }
+
+ currentOffset++;
- //the body of the message is "message" + currentOffset, e.g. "message123"
+ //the body of the message is "message" + currentOffset, e.g. "message123"
- MessageId mid = new MessageId(partitionId, "" + currentOffset, currentOffset);
- EventData ed = new EventData(("message" + currentOffset).getBytes());
- return EventDataWrap.create(ed,mid);
- }
-
- @Override
- public Map<String, Object> getMetricsData() {
- return null;
- }
+ MessageId mid = new MessageId(partitionId, "" + currentOffset, currentOffset);
+ EventData ed = new EventData(("message" + currentOffset).getBytes());
+ return EventDataWrap.create(ed, mid);
+ }
+
+ @Override
+ public Map<String, Object> getMetricsData() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/EventHubSpoutCallerMock.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/EventHubSpoutCallerMock.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/EventHubSpoutCallerMock.java
index 7eb625c..16f3b80 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/EventHubSpoutCallerMock.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/EventHubSpoutCallerMock.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import org.apache.storm.spout.SpoutOutputCollector;
@@ -23,74 +24,69 @@ import org.apache.storm.spout.SpoutOutputCollector;
* Mocks EventHubSpout's caller (storm framework)
*/
public class EventHubSpoutCallerMock {
- public static final String statePathPrefix = "/eventhubspout/TestTopo/namespace/entityname/partitions/";
- EventHubSpout spout;
- private IStateStore stateStore;
- private SpoutOutputCollectorMock collector;
-
- public EventHubSpoutCallerMock(int totalPartitions,
- int totalTasks, int taskIndex, int checkpointInterval) {
- stateStore = new StateStoreMock();
- EventHubSpoutConfig conf = new EventHubSpoutConfig("username", "password",
- "namespace", "entityname", totalPartitions, "zookeeper", checkpointInterval, 1024);
- conf.setTopologyName("TestTopo");
-
- IEventHubReceiverFactory recvFactory = new IEventHubReceiverFactory() {
- @Override
- public IEventHubReceiver create(EventHubSpoutConfig config,
- String partitionId) {
- return new EventHubReceiverMock(partitionId);
- }
- };
- //mock state store and receiver
- spout = new EventHubSpout(conf, stateStore, null, recvFactory);
-
- collector = new SpoutOutputCollectorMock();
-
- try {
- spout.preparePartitions(null, totalTasks, taskIndex, new SpoutOutputCollector(collector));
- }
- catch(Exception ex) {
- }
- }
-
- /**
- * Execute a sequence of calls to EventHubSpout.
- *
- * @param callSequence: is represented as a string of commands,
- * e.g. "r,r,r,r,a1,f2,...". The commands are:
- * r[N]: receive() called N times
- * aP_X: ack(P_X), partition: P, offset: X
- * fP_Y: fail(P_Y), partition: P, offset: Y
- */
- public String execute(String callSequence) {
- String[] cmds = callSequence.split(",");
- for(String cmd : cmds) {
- if(cmd.startsWith("r")) {
- int count = 1;
- if(cmd.length() > 1) {
- count = Integer.parseInt(cmd.substring(1));
+ public static final String statePathPrefix = "/eventhubspout/TestTopo/namespace/entityname/partitions/";
+ EventHubSpout spout;
+ private IStateStore stateStore;
+ private SpoutOutputCollectorMock collector;
+
+ public EventHubSpoutCallerMock(int totalPartitions,
+ int totalTasks, int taskIndex, int checkpointInterval) {
+ stateStore = new StateStoreMock();
+ EventHubSpoutConfig conf = new EventHubSpoutConfig("username", "password",
+ "namespace", "entityname", totalPartitions, "zookeeper", checkpointInterval,
+ 1024);
+ conf.setTopologyName("TestTopo");
+
+ IEventHubReceiverFactory recvFactory = new IEventHubReceiverFactory() {
+ @Override
+ public IEventHubReceiver create(EventHubSpoutConfig config,
+ String partitionId) {
+ return new EventHubReceiverMock(partitionId);
+ }
+ };
+ //mock state store and receiver
+ spout = new EventHubSpout(conf, stateStore, null, recvFactory);
+
+ collector = new SpoutOutputCollectorMock();
+
+ try {
+ spout.preparePartitions(null, totalTasks, taskIndex, new SpoutOutputCollector(collector));
+ } catch (Exception ex) {
}
- for(int i=0; i<count; ++i) {
- spout.nextTuple();
+ }
+
+ /**
+ * Execute a sequence of calls to EventHubSpout.
+ *
+ * @param callSequence: is represented as a string of commands, e.g. "r,r,r,r,a1,f2,...". The commands are: r[N]: receive() called N
+ * times aP_X: ack(P_X), partition: P, offset: X fP_Y: fail(P_Y), partition: P, offset: Y
+ */
+ public String execute(String callSequence) {
+ String[] cmds = callSequence.split(",");
+ for (String cmd : cmds) {
+ if (cmd.startsWith("r")) {
+ int count = 1;
+ if (cmd.length() > 1) {
+ count = Integer.parseInt(cmd.substring(1));
+ }
+ for (int i = 0; i < count; ++i) {
+ spout.nextTuple();
+ }
+ } else if (cmd.startsWith("a")) {
+ String[] midStrs = cmd.substring(1).split("_");
+ MessageId msgId = new MessageId(midStrs[0], midStrs[1], Long.parseLong(midStrs[1]));
+ spout.ack(msgId);
+ } else if (cmd.startsWith("f")) {
+ String[] midStrs = cmd.substring(1).split("_");
+ MessageId msgId = new MessageId(midStrs[0], midStrs[1], Long.parseLong(midStrs[1]));
+ spout.fail(msgId);
+ }
}
- }
- else if(cmd.startsWith("a")) {
- String[] midStrs = cmd.substring(1).split("_");
- MessageId msgId = new MessageId(midStrs[0], midStrs[1], Long.parseLong(midStrs[1]));
- spout.ack(msgId);
- }
- else if(cmd.startsWith("f")) {
- String[] midStrs = cmd.substring(1).split("_");
- MessageId msgId = new MessageId(midStrs[0], midStrs[1], Long.parseLong(midStrs[1]));
- spout.fail(msgId);
- }
+ return collector.getOffsetSequenceAndReset();
+ }
+
+ public String getCheckpoint(int partitionIndex) {
+ String statePath = statePathPrefix + partitionIndex;
+ return stateStore.readData(statePath);
}
- return collector.getOffsetSequenceAndReset();
- }
-
- public String getCheckpoint(int partitionIndex) {
- String statePath = statePathPrefix + partitionIndex;
- return stateStore.readData(statePath);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/PartitionManagerCallerMock.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/PartitionManagerCallerMock.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/PartitionManagerCallerMock.java
index 467461c..364990e 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/PartitionManagerCallerMock.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/PartitionManagerCallerMock.java
@@ -15,87 +15,81 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
/**
* This mock exercises PartitionManager
*/
public class PartitionManagerCallerMock {
- public static final String statePath = "/eventhubspout/TestTopo/namespace/entityname/partitions/1";
- private IPartitionManager pm;
- private IStateStore stateStore;
+ public static final String statePath = "/eventhubspout/TestTopo/namespace/entityname/partitions/1";
+ private IPartitionManager pm;
+ private IStateStore stateStore;
- public PartitionManagerCallerMock(String partitionId) {
- this(partitionId, 0);
- }
-
- public PartitionManagerCallerMock(String partitionId, long enqueueTimeFilter) {
- EventHubReceiverMock receiver = new EventHubReceiverMock(partitionId);
- EventHubSpoutConfig conf = new EventHubSpoutConfig("username", "password",
- "namespace", "entityname", 16, "zookeeper", 10, 1024, 1024, enqueueTimeFilter);
- conf.setTopologyName("TestTopo");
- stateStore = new StateStoreMock();
- this.pm = new PartitionManager(conf, partitionId, stateStore, receiver);
-
- stateStore.open();
- try {
- pm.open();
+ public PartitionManagerCallerMock(String partitionId) {
+ this(partitionId, 0);
}
- catch (Exception ex) {
+
+ public PartitionManagerCallerMock(String partitionId, long enqueueTimeFilter) {
+ EventHubReceiverMock receiver = new EventHubReceiverMock(partitionId);
+ EventHubSpoutConfig conf = new EventHubSpoutConfig("username", "password",
+ "namespace", "entityname", 16, "zookeeper", 10, 1024, 1024, enqueueTimeFilter);
+ conf.setTopologyName("TestTopo");
+ stateStore = new StateStoreMock();
+ this.pm = new PartitionManager(conf, partitionId, stateStore, receiver);
+
+ stateStore.open();
+ try {
+ pm.open();
+ } catch (Exception ex) {
+ }
}
- }
-
- /**
- * Execute a sequence of calls to Partition Manager.
- *
- * @param callSequence: is represented as a string of commands,
- * e.g. "r,r,r,r,a1,f2,...". The commands are:
- * r[N]: receive() called N times
- * aX: ack(X)
- * fY: fail(Y)
- *
- * @return the sequence of messages the receive call returns
- */
- public String execute(String callSequence) {
-
- String[] cmds = callSequence.split(",");
- StringBuilder ret = new StringBuilder();
- for(String cmd : cmds) {
- if(cmd.startsWith("r")) {
- int count = 1;
- if(cmd.length() > 1) {
- count = Integer.parseInt(cmd.substring(1));
+
+ /**
+ * Execute a sequence of calls to Partition Manager.
+ *
+ * @param callSequence: is represented as a string of commands, e.g. "r,r,r,r,a1,f2,...". The commands are: r[N]: receive() called N
+ * times aX: ack(X) fY: fail(Y)
+ * @return the sequence of messages the receive call returns
+ */
+ public String execute(String callSequence) {
+
+ String[] cmds = callSequence.split(",");
+ StringBuilder ret = new StringBuilder();
+ for (String cmd : cmds) {
+ if (cmd.startsWith("r")) {
+ int count = 1;
+ if (cmd.length() > 1) {
+ count = Integer.parseInt(cmd.substring(1));
+ }
+ for (int i = 0; i < count; ++i) {
+ EventDataWrap ed = pm.receive();
+ if (ed == null) {
+ ret.append("null,");
+ } else {
+ ret.append(ed.getMessageId().getOffset());
+ ret.append(",");
+ }
+ }
+ } else if (cmd.startsWith("a")) {
+ pm.ack(cmd.substring(1));
+ } else if (cmd.startsWith("f")) {
+ pm.fail(cmd.substring(1));
+ }
}
- for(int i=0; i<count; ++i) {
- EventDataWrap ed = pm.receive();
- if(ed == null) {
- ret.append("null,");
- }
- else {
- ret.append(ed.getMessageId().getOffset());
- ret.append(",");
- }
+ if (ret.length() > 0) {
+ ret.setLength(ret.length() - 1);
}
- }
- else if(cmd.startsWith("a")) {
- pm.ack(cmd.substring(1));
- }
- else if(cmd.startsWith("f")) {
- pm.fail(cmd.substring(1));
- }
+ return ret.toString();
}
- if(ret.length() > 0) {
- ret.setLength(ret.length()-1);
+
+ /**
+ * Exercise the IPartitionManager.checkpoint() method
+ *
+ * @return the offset that we write to state store
+ */
+ public String checkpoint() {
+ pm.checkpoint();
+ return stateStore.readData(statePath);
}
- return ret.toString();
- }
-
- /**
- * Exercise the IPartitionManager.checkpoint() method
- * @return the offset that we write to state store
- */
- public String checkpoint() {
- pm.checkpoint();
- return stateStore.readData(statePath);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/SpoutOutputCollectorMock.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/SpoutOutputCollectorMock.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/SpoutOutputCollectorMock.java
index 621d6a8..37dfdd5 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/SpoutOutputCollectorMock.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/SpoutOutputCollectorMock.java
@@ -15,57 +15,57 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import java.util.List;
-
import org.apache.storm.spout.ISpoutOutputCollector;
/**
* Mock of ISpoutOutputCollector
*/
public class SpoutOutputCollectorMock implements ISpoutOutputCollector {
- //comma separated offsets
- StringBuilder emittedOffset;
-
- public SpoutOutputCollectorMock() {
- emittedOffset = new StringBuilder();
- }
-
- public String getOffsetSequenceAndReset() {
- String ret = null;
- if(emittedOffset.length() > 0) {
- emittedOffset.setLength(emittedOffset.length()-1);
- ret = emittedOffset.toString();
- emittedOffset.setLength(0);
+ //comma separated offsets
+ StringBuilder emittedOffset;
+
+ public SpoutOutputCollectorMock() {
+ emittedOffset = new StringBuilder();
+ }
+
+ public String getOffsetSequenceAndReset() {
+ String ret = null;
+ if (emittedOffset.length() > 0) {
+ emittedOffset.setLength(emittedOffset.length() - 1);
+ ret = emittedOffset.toString();
+ emittedOffset.setLength(0);
+ }
+ return ret;
}
- return ret;
- }
- @Override
- public List<Integer> emit(String streamId, List<Object> tuple, Object messageId) {
- MessageId mid = (MessageId)messageId;
- String pid = mid.getPartitionId();
- String offset = mid.getOffset();
- emittedOffset.append(pid+"_"+offset+",");
- return null;
- }
+ @Override
+ public List<Integer> emit(String streamId, List<Object> tuple, Object messageId) {
+ MessageId mid = (MessageId) messageId;
+ String pid = mid.getPartitionId();
+ String offset = mid.getOffset();
+ emittedOffset.append(pid + "_" + offset + ",");
+ return null;
+ }
- @Override
- public void emitDirect(int arg0, String arg1, List<Object> arg2, Object arg3) {
- }
+ @Override
+ public void emitDirect(int arg0, String arg1, List<Object> arg2, Object arg3) {
+ }
- @Override
- public void flush() {
- // NO-OP
- }
+ @Override
+ public void flush() {
+ // NO-OP
+ }
- @Override
- public void reportError(Throwable arg0) {
- }
+ @Override
+ public void reportError(Throwable arg0) {
+ }
- @Override
- public long getPendingCount() {
- return 0;
- }
+ @Override
+ public long getPendingCount() {
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/StateStoreMock.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/StateStoreMock.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/StateStoreMock.java
index 0abe3a4..7bfcd93 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/StateStoreMock.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/StateStoreMock.java
@@ -15,40 +15,40 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import java.util.HashMap;
import java.util.Map;
-import org.apache.storm.eventhubs.spout.IStateStore;
-
/**
* A state store mocker
*/
public class StateStoreMock implements IStateStore {
- Map<String, String> myDataMap;
- @Override
- public void open() {
- myDataMap = new HashMap<String, String>();
- }
+ Map<String, String> myDataMap;
- @Override
- public void close() {
- myDataMap = null;
- }
+ @Override
+ public void open() {
+ myDataMap = new HashMap<String, String>();
+ }
+
+ @Override
+ public void close() {
+ myDataMap = null;
+ }
- @Override
- public void saveData(String path, String data) {
- if(myDataMap != null) {
- myDataMap.put(path, data);
+ @Override
+ public void saveData(String path, String data) {
+ if (myDataMap != null) {
+ myDataMap.put(path, data);
+ }
}
- }
- @Override
- public String readData(String path) {
- if(myDataMap != null) {
- return myDataMap.get(path);
+ @Override
+ public String readData(String path) {
+ if (myDataMap != null) {
+ return myDataMap.get(path);
+ }
+ return null;
}
- return null;
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestEventData.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestEventData.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestEventData.java
index 926337b..ab54f66 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestEventData.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestEventData.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import org.junit.After;
@@ -25,23 +26,23 @@ import static org.junit.Assert.assertTrue;
public class TestEventData {
- @Before
- public void setUp() throws Exception {
- }
+ @Before
+ public void setUp() throws Exception {
+ }
- @After
- public void tearDown() throws Exception {
- }
+ @After
+ public void tearDown() throws Exception {
+ }
- @Test
- public void testEventDataComparision() {
+ @Test
+ public void testEventDataComparision() {
- MessageId messageId1 = MessageId.create(null, "3", 1);
- EventDataWrap eventData1 = EventDataWrap.create(null, messageId1);
+ MessageId messageId1 = MessageId.create(null, "3", 1);
+ EventDataWrap eventData1 = EventDataWrap.create(null, messageId1);
- MessageId messageId2 = MessageId.create(null, "13", 2);
- EventDataWrap eventData2 = EventDataWrap.create(null, messageId2);
+ MessageId messageId2 = MessageId.create(null, "13", 2);
+ EventDataWrap eventData2 = EventDataWrap.create(null, messageId2);
- assertTrue(eventData2.compareTo(eventData1) > 0);
- }
+ assertTrue(eventData2.compareTo(eventData1) > 0);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestEventHubSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestEventHubSpout.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestEventHubSpout.java
index a7b3588..06f3574 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestEventHubSpout.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestEventHubSpout.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import org.junit.After;
@@ -24,50 +25,51 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
-
public class TestEventHubSpout {
- @Before
- public void setUp() throws Exception {
- }
+ @Before
+ public void setUp() throws Exception {
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public void testSpoutConfig() {
+ EventHubSpoutConfig conf = new EventHubSpoutConfig("username", "pas\\s+w/ord",
+ "namespace", "entityname", 16);
+ conf.setZkConnectionString("zookeeper");
+ conf.setCheckpointIntervalInSeconds(1);
+ assertEquals(conf.getConnectionString(),
+ "Endpoint=amqps://namespace.servicebus.windows.net;EntityPath=entityname;SharedAccessKeyName=username;" +
+ "SharedAccessKey=pas\\s+w/ord;OperationTimeout=PT1M;RetryPolicy=Default");
+ }
+
+ @Test
+ public void testSpoutBasic() {
+ //This spout owns 2 partitions: 6 and 14
+ EventHubSpoutCallerMock mock = new EventHubSpoutCallerMock(16, 8, 6, 10);
+ String result = mock.execute("r6,f6_0,a6_1,a6_2,a14_0,a14_2,r4,f14_1,r2");
+ assertEquals("6_0,14_0,6_1,14_1,6_2,14_2,6_0,14_3,6_3,14_4,6_4,14_1", result);
+ }
- @After
- public void tearDown() throws Exception {
- }
-
- @Test
- public void testSpoutConfig() {
- EventHubSpoutConfig conf = new EventHubSpoutConfig("username", "pas\\s+w/ord",
- "namespace", "entityname", 16);
- conf.setZkConnectionString("zookeeper");
- conf.setCheckpointIntervalInSeconds(1);
- assertEquals(conf.getConnectionString(), "Endpoint=amqps://namespace.servicebus.windows.net;EntityPath=entityname;SharedAccessKeyName=username;SharedAccessKey=pas\\s+w/ord;OperationTimeout=PT1M;RetryPolicy=Default");
- }
+ @Test
+ public void testSpoutCheckpoint() {
+ //Make sure that even though nextTuple() doesn't receive valid data,
+ //the offset will be checkpointed after checkpointInterval seconds.
- @Test
- public void testSpoutBasic() {
- //This spout owns 2 partitions: 6 and 14
- EventHubSpoutCallerMock mock = new EventHubSpoutCallerMock(16, 8, 6,10);
- String result = mock.execute("r6,f6_0,a6_1,a6_2,a14_0,a14_2,r4,f14_1,r2");
- assertEquals("6_0,14_0,6_1,14_1,6_2,14_2,6_0,14_3,6_3,14_4,6_4,14_1", result);
- }
-
- @Test
- public void testSpoutCheckpoint() {
- //Make sure that even though nextTuple() doesn't receive valid data,
- //the offset will be checkpointed after checkpointInterval seconds.
-
- //This spout owns 1 partitions: 6
- EventHubSpoutCallerMock mock = new EventHubSpoutCallerMock(8, 8, 6, 1);
- String result = mock.execute("r6,a6_0,a6_1,a6_2");
- try {
- Thread.sleep(2000);
+ //This spout owns 1 partitions: 6
+ EventHubSpoutCallerMock mock = new EventHubSpoutCallerMock(8, 8, 6, 1);
+ String result = mock.execute("r6,a6_0,a6_1,a6_2");
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException ex) {
+ }
+ EventHubReceiverMock.setPause(true);
+ result = mock.execute("r3");
+ EventHubReceiverMock.setPause(false);
+ assertEquals("3", mock.getCheckpoint(6));
}
- catch(InterruptedException ex) {}
- EventHubReceiverMock.setPause(true);
- result = mock.execute("r3");
- EventHubReceiverMock.setPause(false);
- assertEquals("3", mock.getCheckpoint(6));
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestPartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestPartitionManager.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestPartitionManager.java
index 5b28f1d..e4b16ef 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestPartitionManager.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/spout/TestPartitionManager.java
@@ -15,103 +15,105 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.spout;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.*;
+
+import static org.junit.Assert.assertEquals;
public class TestPartitionManager {
- @Before
- public void setUp() throws Exception {
- }
+ @Before
+ public void setUp() throws Exception {
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public void testPartitionManagerNoFail() {
+ PartitionManagerCallerMock mock
+ = new PartitionManagerCallerMock("1");
+ String result = mock.execute("r,r,r,a0,a1,a2,r");
+ assertEquals("0,1,2,3", result);
+ }
+
+ @Test
+ public void testPartitionManagerResend() {
+ PartitionManagerCallerMock mock
+ = new PartitionManagerCallerMock("1");
+ String result = mock.execute("r,a0,r,r,r,f3,r,f2,f1,r,r,a1,a2,a3,r");
+ assertEquals("0,1,2,3,3,1,2,4", result);
+ }
+
+ @Test
+ public void testPMCheckpointWithPending() {
+ PartitionManagerCallerMock mock
+ = new PartitionManagerCallerMock("1");
+ mock.execute("r,r,r");
+ //no ack, so return the first of pending list
+ assertEquals("0", mock.checkpoint());
+ mock.execute("a0,a2");
+ //still need to return the first of pending list
+ assertEquals("1", mock.checkpoint());
+ }
+
+ @Test
+ public void testPMCheckpointWithResend() {
+ PartitionManagerCallerMock mock
+ = new PartitionManagerCallerMock("1");
+ mock.execute("r,r,r,f2,f1,f0");
+ //pending is empty, return the smallest in toResend
+ assertEquals("0", mock.checkpoint());
+ mock.execute("r,a0");
+ //pending is still empty
+ assertEquals("1", mock.checkpoint());
+ }
+
+ @Test
+ public void testPMCheckpointWithPendingAndResend() {
+ PartitionManagerCallerMock mock
+ = new PartitionManagerCallerMock("1");
+ mock.execute("r,r,r,f2,f1");
+ //return the smaller of pending and toResend
+ assertEquals("0", mock.checkpoint());
+ mock.execute("a0,r");
+ //now pending: [3], toResend: [1,2]
+ assertEquals("1", mock.checkpoint());
+ }
- @After
- public void tearDown() throws Exception {
- }
+ @Test
+ public void testPMCheckpointWithNoPendingAndNoResend() {
+ PartitionManagerCallerMock mock
+ = new PartitionManagerCallerMock("1");
+ //if no event sent, no checkpoint shall be created
+ assertEquals(null, mock.checkpoint());
+ mock.execute("r,r,r,f2,f1,r,r,a2,a1,a0");
+ //all events are sent successfully, return last sent offset
+ assertEquals("2", mock.checkpoint());
+ }
- @Test
- public void testPartitionManagerNoFail() {
- PartitionManagerCallerMock mock
- = new PartitionManagerCallerMock("1");
- String result = mock.execute("r,r,r,a0,a1,a2,r");
- assertEquals("0,1,2,3", result);
- }
+ @Test
+ public void testPartitionManagerMaxPendingMessages() {
+ PartitionManagerCallerMock mock
+ = new PartitionManagerCallerMock("1");
+ String result = mock.execute("r1024");
+ //any receive call after exceeding max pending messages results in null
+ result = mock.execute("r2");
+ assertEquals("null,null", result);
+ result = mock.execute("a0,a1,r2");
+ assertEquals("1024,1025", result);
+ }
- @Test
- public void testPartitionManagerResend() {
- PartitionManagerCallerMock mock
- = new PartitionManagerCallerMock("1");
- String result = mock.execute("r,a0,r,r,r,f3,r,f2,f1,r,r,a1,a2,a3,r");
- assertEquals("0,1,2,3,3,1,2,4", result);
- }
-
- @Test
- public void testPMCheckpointWithPending() {
- PartitionManagerCallerMock mock
- = new PartitionManagerCallerMock("1");
- mock.execute("r,r,r");
- //no ack, so return the first of pending list
- assertEquals("0", mock.checkpoint());
- mock.execute("a0,a2");
- //still need to return the first of pending list
- assertEquals("1", mock.checkpoint());
- }
-
- @Test
- public void testPMCheckpointWithResend() {
- PartitionManagerCallerMock mock
- = new PartitionManagerCallerMock("1");
- mock.execute("r,r,r,f2,f1,f0");
- //pending is empty, return the smallest in toResend
- assertEquals("0", mock.checkpoint());
- mock.execute("r,a0");
- //pending is still empty
- assertEquals("1", mock.checkpoint());
- }
-
- @Test
- public void testPMCheckpointWithPendingAndResend() {
- PartitionManagerCallerMock mock
- = new PartitionManagerCallerMock("1");
- mock.execute("r,r,r,f2,f1");
- //return the smaller of pending and toResend
- assertEquals("0", mock.checkpoint());
- mock.execute("a0,r");
- //now pending: [3], toResend: [1,2]
- assertEquals("1", mock.checkpoint());
- }
-
- @Test
- public void testPMCheckpointWithNoPendingAndNoResend() {
- PartitionManagerCallerMock mock
- = new PartitionManagerCallerMock("1");
- //if no event sent, no checkpoint shall be created
- assertEquals(null, mock.checkpoint());
- mock.execute("r,r,r,f2,f1,r,r,a2,a1,a0");
- //all events are sent successfully, return last sent offset
- assertEquals("2", mock.checkpoint());
- }
-
- @Test
- public void testPartitionManagerMaxPendingMessages() {
- PartitionManagerCallerMock mock
- = new PartitionManagerCallerMock("1");
- String result = mock.execute("r1024");
- //any receive call after exceeding max pending messages results in null
- result = mock.execute("r2");
- assertEquals("null,null", result);
- result = mock.execute("a0,a1,r2");
- assertEquals("1024,1025", result);
- }
-
- @Test
- public void testPartitionManagerEnqueueTimeFilter() {
- PartitionManagerCallerMock mock
- = new PartitionManagerCallerMock("1", 123456);
- String result = mock.execute("r2");
- assertEquals("123457,123458", result);
- }
+ @Test
+ public void testPartitionManagerEnqueueTimeFilter() {
+ PartitionManagerCallerMock mock
+ = new PartitionManagerCallerMock("1", 123456);
+ String result = mock.execute("r2");
+ assertEquals("123457,123458", result);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/trident/TestTransactionalTridentEmitter.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/trident/TestTransactionalTridentEmitter.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/trident/TestTransactionalTridentEmitter.java
index b6ce76a..31cbc32 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/trident/TestTransactionalTridentEmitter.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/trident/TestTransactionalTridentEmitter.java
@@ -15,79 +15,78 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
-package org.apache.storm.eventhubs.trident;
-import static org.junit.Assert.*;
+package org.apache.storm.eventhubs.trident;
import java.util.Map;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
import org.apache.storm.eventhubs.spout.EventHubReceiverMock;
import org.apache.storm.eventhubs.spout.EventHubSpoutConfig;
import org.apache.storm.eventhubs.spout.IEventHubReceiver;
import org.apache.storm.eventhubs.spout.IEventHubReceiverFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
public class TestTransactionalTridentEmitter {
- private TransactionalTridentEventHubEmitter emitter;
- private Partition partition;
- private TridentCollectorMock collectorMock;
- private final int batchSize = 32;
+ private final int batchSize = 32;
+ private TransactionalTridentEventHubEmitter emitter;
+ private Partition partition;
+ private TridentCollectorMock collectorMock;
+
+ @Before
+ public void setUp() throws Exception {
+ EventHubSpoutConfig conf = new EventHubSpoutConfig("username", "password",
+ "namespace", "entityname", 16, "zookeeper");
+ conf.setTopologyName("TestTopo");
+ IEventHubReceiverFactory recvFactory = new IEventHubReceiverFactory() {
+ @Override
+ public IEventHubReceiver create(EventHubSpoutConfig config,
+ String partitionId) {
+ return new EventHubReceiverMock(partitionId);
+ }
+ };
+ partition = new Partition(conf, "0");
+ emitter = new TransactionalTridentEventHubEmitter(conf, batchSize, null, recvFactory);
+ collectorMock = new TridentCollectorMock();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ emitter.close();
+ emitter = null;
+ }
+
+ @Test
+ public void testEmitInSequence() {
+ //test the happy path, emit batches in sequence
+ Map<String, Object> meta = emitter.emitPartitionBatchNew(null, collectorMock, partition, null);
+ String collected = collectorMock.getBuffer();
+ assertTrue(collected.startsWith("message" + 0));
+ //System.out.println("collected: " + collected);
+ collectorMock.clear();
- @Before
- public void setUp() throws Exception {
- EventHubSpoutConfig conf = new EventHubSpoutConfig("username", "password",
- "namespace", "entityname", 16, "zookeeper");
- conf.setTopologyName("TestTopo");
- IEventHubReceiverFactory recvFactory = new IEventHubReceiverFactory() {
- @Override
- public IEventHubReceiver create(EventHubSpoutConfig config,
- String partitionId) {
- return new EventHubReceiverMock(partitionId);
- }
- };
- partition = new Partition(conf, "0");
- emitter = new TransactionalTridentEventHubEmitter(conf, batchSize, null, recvFactory);
- collectorMock = new TridentCollectorMock();
- }
+ emitter.emitPartitionBatchNew(null, collectorMock, partition, meta);
+ collected = collectorMock.getBuffer();
+ //System.out.println("collected: " + collected);
+ assertTrue(collected.startsWith("message" + batchSize));
+ }
- @After
- public void tearDown() throws Exception {
- emitter.close();
- emitter = null;
- }
+ @Test
+ public void testReEmit() {
+ //test we can re-emit the second batch
+ Map<String, Object> meta = emitter.emitPartitionBatchNew(null, collectorMock, partition, null);
+ collectorMock.clear();
- @Test
- public void testEmitInSequence() {
- //test the happy path, emit batches in sequence
- Map<String, Object> meta = emitter.emitPartitionBatchNew(null, collectorMock, partition, null);
- String collected = collectorMock.getBuffer();
- assertTrue(collected.startsWith("message"+0));
- //System.out.println("collected: " + collected);
- collectorMock.clear();
-
- emitter.emitPartitionBatchNew(null, collectorMock, partition, meta);
- collected = collectorMock.getBuffer();
- //System.out.println("collected: " + collected);
- assertTrue(collected.startsWith("message"+batchSize));
- }
+ //emit second batch
+ Map<String, Object> meta1 = emitter.emitPartitionBatchNew(null, collectorMock, partition, meta);
+ String collected0 = collectorMock.getBuffer();
+ collectorMock.clear();
- @Test
- public void testReEmit() {
- //test we can re-emit the second batch
- Map<String, Object> meta = emitter.emitPartitionBatchNew(null, collectorMock, partition, null);
- collectorMock.clear();
-
- //emit second batch
- Map<String, Object> meta1 = emitter.emitPartitionBatchNew(null, collectorMock, partition, meta);
- String collected0 = collectorMock.getBuffer();
- collectorMock.clear();
-
- //re-emit second batch
- emitter.emitPartitionBatch(null, collectorMock, partition, meta1);
- String collected1 = collectorMock.getBuffer();
- assertTrue(collected0.equals(collected1));
- }
+ //re-emit second batch
+ emitter.emitPartitionBatch(null, collectorMock, partition, meta1);
+ String collected1 = collectorMock.getBuffer();
+ assertTrue(collected0.equals(collected1));
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/18723171/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/trident/TridentCollectorMock.java
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/trident/TridentCollectorMock.java b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/trident/TridentCollectorMock.java
index bd5b07f..59f7002 100755
--- a/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/trident/TridentCollectorMock.java
+++ b/external/storm-eventhubs/src/test/java/org/apache/storm/eventhubs/trident/TridentCollectorMock.java
@@ -15,43 +15,43 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/
+
package org.apache.storm.eventhubs.trident;
import java.util.List;
-
import org.apache.storm.trident.operation.TridentCollector;
/**
* A mock of TridentCollector
*/
public class TridentCollectorMock implements TridentCollector {
- StringBuilder buffer;
-
- public TridentCollectorMock() {
- buffer = new StringBuilder();
- }
-
- @Override
- public void emit(List<Object> tuples) {
- for(Object o: tuples) {
- buffer.append(o.toString());
+ StringBuilder buffer;
+
+ public TridentCollectorMock() {
+ buffer = new StringBuilder();
+ }
+
+ @Override
+ public void emit(List<Object> tuples) {
+ for (Object o : tuples) {
+ buffer.append(o.toString());
+ }
+ }
+
+ @Override
+ public void flush() {
+ // NO-OP
+ }
+
+ @Override
+ public void reportError(Throwable arg0) {
+ }
+
+ public void clear() {
+ buffer.setLength(0);
+ }
+
+ public String getBuffer() {
+ return buffer.toString();
}
- }
-
- @Override
- public void flush() {
- // NO-OP
- }
-
- @Override
- public void reportError(Throwable arg0) {
- }
-
- public void clear() {
- buffer.setLength(0);
- }
-
- public String getBuffer() {
- return buffer.toString();
- }
}
[38/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-starter
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java
index 23326c5..2580dfb 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBoltTest.java
@@ -1,22 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.bolt;
+import com.google.common.collect.Lists;
+import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -24,123 +21,124 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.MockTupleHelpers;
-import com.google.common.collect.Lists;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import java.util.Map;
-
import static org.fest.assertions.api.Assertions.assertThat;
import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
public class IntermediateRankingsBoltTest {
- private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id";
- private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id";
- private static final Object ANY_OBJECT = new Object();
- private static final int ANY_TOPN = 10;
- private static final long ANY_COUNT = 42;
-
- private Tuple mockRankableTuple(Object obj, long count) {
- Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID);
- when(tuple.getValues()).thenReturn(Lists.newArrayList(ANY_OBJECT, ANY_COUNT));
- return tuple;
- }
-
- @DataProvider
- public Object[][] illegalTopN() {
- return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopN")
- public void negativeOrZeroTopNShouldThrowIAE(int topN) {
- new IntermediateRankingsBolt(topN);
- }
-
- @DataProvider
- public Object[][] illegalEmitFrequency() {
- return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency")
- public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) {
- new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds);
- }
-
- @DataProvider
- public Object[][] legalTopN() {
- return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
- }
-
- @Test(dataProvider = "legalTopN")
- public void positiveTopNShouldBeOk(int topN) {
- new IntermediateRankingsBolt(topN);
- }
-
- @DataProvider
- public Object[][] legalEmitFrequency() {
- return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
- }
-
- @Test(dataProvider = "legalEmitFrequency")
- public void positiveEmitFrequencyShouldBeOk(int emitFrequencyInSeconds) {
- new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds);
- }
-
- @Test
- public void shouldEmitSomethingIfTickTupleIsReceived() {
- // given
- Tuple tickTuple = MockTupleHelpers.mockTickTuple();
- BasicOutputCollector collector = mock(BasicOutputCollector.class);
- IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();
-
- // when
- bolt.execute(tickTuple, collector);
-
- // then
- // verifyZeroInteractions(collector);
- verify(collector).emit(any(Values.class));
- }
-
- @Test
- public void shouldEmitNothingIfNormalTupleIsReceived() {
- // given
- Tuple normalTuple = mockRankableTuple(ANY_OBJECT, ANY_COUNT);
- BasicOutputCollector collector = mock(BasicOutputCollector.class);
- IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();
-
- // when
- bolt.execute(normalTuple, collector);
-
- // then
- verifyZeroInteractions(collector);
- }
-
- @Test
- public void shouldDeclareOutputFields() {
- // given
- OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class);
- IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();
-
- // when
- bolt.declareOutputFields(declarer);
-
- // then
- verify(declarer, times(1)).declare(any(Fields.class));
- }
-
- @Test
- public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() {
- // given
- IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();
-
- // when
- Map<String, Object> componentConfig = bolt.getComponentConfiguration();
-
- // then
- assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
- Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
- assertThat(emitFrequencyInSeconds).isGreaterThan(0);
- }
+ private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id";
+ private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id";
+ private static final Object ANY_OBJECT = new Object();
+ private static final int ANY_TOPN = 10;
+ private static final long ANY_COUNT = 42;
+
+ private Tuple mockRankableTuple(Object obj, long count) {
+ Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID);
+ when(tuple.getValues()).thenReturn(Lists.newArrayList(ANY_OBJECT, ANY_COUNT));
+ return tuple;
+ }
+
+ @DataProvider
+ public Object[][] illegalTopN() {
+ return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopN")
+ public void negativeOrZeroTopNShouldThrowIAE(int topN) {
+ new IntermediateRankingsBolt(topN);
+ }
+
+ @DataProvider
+ public Object[][] illegalEmitFrequency() {
+ return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency")
+ public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) {
+ new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds);
+ }
+
+ @DataProvider
+ public Object[][] legalTopN() {
+ return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
+ }
+
+ @Test(dataProvider = "legalTopN")
+ public void positiveTopNShouldBeOk(int topN) {
+ new IntermediateRankingsBolt(topN);
+ }
+
+ @DataProvider
+ public Object[][] legalEmitFrequency() {
+ return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
+ }
+
+ @Test(dataProvider = "legalEmitFrequency")
+ public void positiveEmitFrequencyShouldBeOk(int emitFrequencyInSeconds) {
+ new IntermediateRankingsBolt(ANY_TOPN, emitFrequencyInSeconds);
+ }
+
+ @Test
+ public void shouldEmitSomethingIfTickTupleIsReceived() {
+ // given
+ Tuple tickTuple = MockTupleHelpers.mockTickTuple();
+ BasicOutputCollector collector = mock(BasicOutputCollector.class);
+ IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();
+
+ // when
+ bolt.execute(tickTuple, collector);
+
+ // then
+ // verifyZeroInteractions(collector);
+ verify(collector).emit(any(Values.class));
+ }
+
+ @Test
+ public void shouldEmitNothingIfNormalTupleIsReceived() {
+ // given
+ Tuple normalTuple = mockRankableTuple(ANY_OBJECT, ANY_COUNT);
+ BasicOutputCollector collector = mock(BasicOutputCollector.class);
+ IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();
+
+ // when
+ bolt.execute(normalTuple, collector);
+
+ // then
+ verifyZeroInteractions(collector);
+ }
+
+ @Test
+ public void shouldDeclareOutputFields() {
+ // given
+ OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class);
+ IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();
+
+ // when
+ bolt.declareOutputFields(declarer);
+
+ // then
+ verify(declarer, times(1)).declare(any(Fields.class));
+ }
+
+ @Test
+ public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() {
+ // given
+ IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();
+
+ // when
+ Map<String, Object> componentConfig = bolt.getComponentConfiguration();
+
+ // then
+ assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
+ Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
+ assertThat(emitFrequencyInSeconds).isGreaterThan(0);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java
index e58c356..ccac7bc 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/RollingCountBoltTest.java
@@ -1,22 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.bolt;
+import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -27,87 +23,89 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.MockTupleHelpers;
import org.testng.annotations.Test;
-import java.util.Map;
-
import static org.fest.assertions.api.Assertions.assertThat;
import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
public class RollingCountBoltTest {
- private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id";
- private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id";
-
- private Tuple mockNormalTuple(Object obj) {
- Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID);
- when(tuple.getValue(0)).thenReturn(obj);
- return tuple;
- }
-
- @SuppressWarnings("rawtypes")
- @Test
- public void shouldEmitNothingIfNoObjectHasBeenCountedYetAndTickTupleIsReceived() {
- // given
- Tuple tickTuple = MockTupleHelpers.mockTickTuple();
- RollingCountBolt bolt = new RollingCountBolt();
- Map<String, Object> conf = mock(Map.class);
- TopologyContext context = mock(TopologyContext.class);
- OutputCollector collector = mock(OutputCollector.class);
- bolt.prepare(conf, context, collector);
-
- // when
- bolt.execute(tickTuple);
-
- // then
- verifyZeroInteractions(collector);
- }
-
- @SuppressWarnings("rawtypes")
- @Test
- public void shouldEmitSomethingIfAtLeastOneObjectWasCountedAndTickTupleIsReceived() {
- // given
- Tuple normalTuple = mockNormalTuple(new Object());
- Tuple tickTuple = MockTupleHelpers.mockTickTuple();
-
- RollingCountBolt bolt = new RollingCountBolt();
- Map<String, Object> conf = mock(Map.class);
- TopologyContext context = mock(TopologyContext.class);
- OutputCollector collector = mock(OutputCollector.class);
- bolt.prepare(conf, context, collector);
-
- // when
- bolt.execute(normalTuple);
- bolt.execute(tickTuple);
-
- // then
- verify(collector).emit(any(Values.class));
- }
-
- @Test
- public void shouldDeclareOutputFields() {
- // given
- OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class);
- RollingCountBolt bolt = new RollingCountBolt();
-
- // when
- bolt.declareOutputFields(declarer);
-
- // then
- verify(declarer, times(1)).declare(any(Fields.class));
-
- }
-
- @Test
- public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() {
- // given
- RollingCountBolt bolt = new RollingCountBolt();
-
- // when
- Map<String, Object> componentConfig = bolt.getComponentConfiguration();
-
- // then
- assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
- Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
- assertThat(emitFrequencyInSeconds).isGreaterThan(0);
- }
+ private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id";
+ private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id";
+
+ private Tuple mockNormalTuple(Object obj) {
+ Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID);
+ when(tuple.getValue(0)).thenReturn(obj);
+ return tuple;
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void shouldEmitNothingIfNoObjectHasBeenCountedYetAndTickTupleIsReceived() {
+ // given
+ Tuple tickTuple = MockTupleHelpers.mockTickTuple();
+ RollingCountBolt bolt = new RollingCountBolt();
+ Map<String, Object> conf = mock(Map.class);
+ TopologyContext context = mock(TopologyContext.class);
+ OutputCollector collector = mock(OutputCollector.class);
+ bolt.prepare(conf, context, collector);
+
+ // when
+ bolt.execute(tickTuple);
+
+ // then
+ verifyZeroInteractions(collector);
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Test
+ public void shouldEmitSomethingIfAtLeastOneObjectWasCountedAndTickTupleIsReceived() {
+ // given
+ Tuple normalTuple = mockNormalTuple(new Object());
+ Tuple tickTuple = MockTupleHelpers.mockTickTuple();
+
+ RollingCountBolt bolt = new RollingCountBolt();
+ Map<String, Object> conf = mock(Map.class);
+ TopologyContext context = mock(TopologyContext.class);
+ OutputCollector collector = mock(OutputCollector.class);
+ bolt.prepare(conf, context, collector);
+
+ // when
+ bolt.execute(normalTuple);
+ bolt.execute(tickTuple);
+
+ // then
+ verify(collector).emit(any(Values.class));
+ }
+
+ @Test
+ public void shouldDeclareOutputFields() {
+ // given
+ OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class);
+ RollingCountBolt bolt = new RollingCountBolt();
+
+ // when
+ bolt.declareOutputFields(declarer);
+
+ // then
+ verify(declarer, times(1)).declare(any(Fields.class));
+
+ }
+
+ @Test
+ public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() {
+ // given
+ RollingCountBolt bolt = new RollingCountBolt();
+
+ // when
+ Map<String, Object> componentConfig = bolt.getComponentConfiguration();
+
+ // then
+ assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
+ Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
+ assertThat(emitFrequencyInSeconds).isGreaterThan(0);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java
index c3582d5..bc3ca1f 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/bolt/TotalRankingsBoltTest.java
@@ -1,23 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.bolt;
+import java.util.Map;
import org.apache.storm.Config;
+import org.apache.storm.starter.tools.Rankings;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
@@ -26,122 +23,123 @@ import org.apache.storm.tuple.Values;
import org.apache.storm.utils.MockTupleHelpers;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import org.apache.storm.starter.tools.Rankings;
-
-import java.util.Map;
import static org.fest.assertions.api.Assertions.assertThat;
import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
public class TotalRankingsBoltTest {
- private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id";
- private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id";
- private static final Object ANY_OBJECT = new Object();
- private static final int ANY_TOPN = 10;
- private static final long ANY_COUNT = 42;
-
- private Tuple mockRankingsTuple(Object obj, long count) {
- Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID);
- Rankings rankings = mock(Rankings.class);
- when(tuple.getValue(0)).thenReturn(rankings);
- return tuple;
- }
-
- @DataProvider
- public Object[][] illegalTopN() {
- return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopN")
- public void negativeOrZeroTopNShouldThrowIAE(int topN) {
- new TotalRankingsBolt(topN);
- }
-
- @DataProvider
- public Object[][] illegalEmitFrequency() {
- return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency")
- public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) {
- new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds);
- }
-
- @DataProvider
- public Object[][] legalTopN() {
- return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
- }
-
- @Test(dataProvider = "legalTopN")
- public void positiveTopNShouldBeOk(int topN) {
- new TotalRankingsBolt(topN);
- }
-
- @DataProvider
- public Object[][] legalEmitFrequency() {
- return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
- }
-
- @Test(dataProvider = "legalEmitFrequency")
- public void positiveEmitFrequencyShouldBeOk(int emitFrequencyInSeconds) {
- new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds);
- }
-
- @Test
- public void shouldEmitSomethingIfTickTupleIsReceived() {
- // given
- Tuple tickTuple = MockTupleHelpers.mockTickTuple();
- BasicOutputCollector collector = mock(BasicOutputCollector.class);
- TotalRankingsBolt bolt = new TotalRankingsBolt();
-
- // when
- bolt.execute(tickTuple, collector);
-
- // then
- // verifyZeroInteractions(collector);
- verify(collector).emit(any(Values.class));
- }
-
- @Test
- public void shouldEmitNothingIfNormalTupleIsReceived() {
- // given
- Tuple normalTuple = mockRankingsTuple(ANY_OBJECT, ANY_COUNT);
- BasicOutputCollector collector = mock(BasicOutputCollector.class);
- TotalRankingsBolt bolt = new TotalRankingsBolt();
-
- // when
- bolt.execute(normalTuple, collector);
-
- // then
- verifyZeroInteractions(collector);
- }
-
- @Test
- public void shouldDeclareOutputFields() {
- // given
- OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class);
- TotalRankingsBolt bolt = new TotalRankingsBolt();
-
- // when
- bolt.declareOutputFields(declarer);
-
- // then
- verify(declarer, times(1)).declare(any(Fields.class));
- }
-
- @Test
- public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() {
- // given
- TotalRankingsBolt bolt = new TotalRankingsBolt();
-
- // when
- Map<String, Object> componentConfig = bolt.getComponentConfiguration();
-
- // then
- assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
- Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
- assertThat(emitFrequencyInSeconds).isGreaterThan(0);
- }
+ private static final String ANY_NON_SYSTEM_COMPONENT_ID = "irrelevant_component_id";
+ private static final String ANY_NON_SYSTEM_STREAM_ID = "irrelevant_stream_id";
+ private static final Object ANY_OBJECT = new Object();
+ private static final int ANY_TOPN = 10;
+ private static final long ANY_COUNT = 42;
+
+ private Tuple mockRankingsTuple(Object obj, long count) {
+ Tuple tuple = MockTupleHelpers.mockTuple(ANY_NON_SYSTEM_COMPONENT_ID, ANY_NON_SYSTEM_STREAM_ID);
+ Rankings rankings = mock(Rankings.class);
+ when(tuple.getValue(0)).thenReturn(rankings);
+ return tuple;
+ }
+
+ @DataProvider
+ public Object[][] illegalTopN() {
+ return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopN")
+ public void negativeOrZeroTopNShouldThrowIAE(int topN) {
+ new TotalRankingsBolt(topN);
+ }
+
+ @DataProvider
+ public Object[][] illegalEmitFrequency() {
+ return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalEmitFrequency")
+ public void negativeOrZeroEmitFrequencyShouldThrowIAE(int emitFrequencyInSeconds) {
+ new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds);
+ }
+
+ @DataProvider
+ public Object[][] legalTopN() {
+ return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
+ }
+
+ @Test(dataProvider = "legalTopN")
+ public void positiveTopNShouldBeOk(int topN) {
+ new TotalRankingsBolt(topN);
+ }
+
+ @DataProvider
+ public Object[][] legalEmitFrequency() {
+ return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
+ }
+
+ @Test(dataProvider = "legalEmitFrequency")
+ public void positiveEmitFrequencyShouldBeOk(int emitFrequencyInSeconds) {
+ new TotalRankingsBolt(ANY_TOPN, emitFrequencyInSeconds);
+ }
+
+ @Test
+ public void shouldEmitSomethingIfTickTupleIsReceived() {
+ // given
+ Tuple tickTuple = MockTupleHelpers.mockTickTuple();
+ BasicOutputCollector collector = mock(BasicOutputCollector.class);
+ TotalRankingsBolt bolt = new TotalRankingsBolt();
+
+ // when
+ bolt.execute(tickTuple, collector);
+
+ // then
+ // verifyZeroInteractions(collector);
+ verify(collector).emit(any(Values.class));
+ }
+
+ @Test
+ public void shouldEmitNothingIfNormalTupleIsReceived() {
+ // given
+ Tuple normalTuple = mockRankingsTuple(ANY_OBJECT, ANY_COUNT);
+ BasicOutputCollector collector = mock(BasicOutputCollector.class);
+ TotalRankingsBolt bolt = new TotalRankingsBolt();
+
+ // when
+ bolt.execute(normalTuple, collector);
+
+ // then
+ verifyZeroInteractions(collector);
+ }
+
+ @Test
+ public void shouldDeclareOutputFields() {
+ // given
+ OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class);
+ TotalRankingsBolt bolt = new TotalRankingsBolt();
+
+ // when
+ bolt.declareOutputFields(declarer);
+
+ // then
+ verify(declarer, times(1)).declare(any(Fields.class));
+ }
+
+ @Test
+ public void shouldSetTickTupleFrequencyInComponentConfigurationToNonZeroValue() {
+ // given
+ TotalRankingsBolt bolt = new TotalRankingsBolt();
+
+ // when
+ Map<String, Object> componentConfig = bolt.getComponentConfiguration();
+
+ // then
+ assertThat(componentConfig).containsKey(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
+ Integer emitFrequencyInSeconds = (Integer) componentConfig.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
+ assertThat(emitFrequencyInSeconds).isGreaterThan(0);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java
index 2c07168..f4393ed 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTrackerTest.java
@@ -1,102 +1,98 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.tools;
import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Time.SimulatedTime;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static org.fest.assertions.api.Assertions.assertThat;
-import org.apache.storm.utils.Time.SimulatedTime;
-
public class NthLastModifiedTimeTrackerTest {
- private static final int ANY_NUM_TIMES_TO_TRACK = 3;
- private static final int MILLIS_IN_SEC = 1000;
-
- @DataProvider
- public Object[][] illegalNumTimesData() {
- return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalNumTimesData")
- public void negativeOrZeroNumTimesToTrackShouldThrowIAE(int numTimesToTrack) {
- new NthLastModifiedTimeTracker(numTimesToTrack);
- }
-
- @DataProvider
- public Object[][] legalNumTimesData() {
- return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
- }
-
- @Test(dataProvider = "legalNumTimesData")
- public void positiveNumTimesToTrackShouldBeOk(int numTimesToTrack) {
- new NthLastModifiedTimeTracker(numTimesToTrack);
- }
-
- @DataProvider
- public Object[][] whenNotYetMarkedAsModifiedData() {
- return new Object[][]{ { 0 }, { 1 }, { 2 }, { 3 }, { 4 }, { 5 }, { 8 }, { 10 } };
- }
-
- @Test(dataProvider = "whenNotYetMarkedAsModifiedData")
- public void shouldReturnCorrectModifiedTimeEvenWhenNotYetMarkedAsModified(int secondsToAdvance) {
- // given
- try (SimulatedTime t = new SimulatedTime()) {
- NthLastModifiedTimeTracker tracker = new NthLastModifiedTimeTracker(ANY_NUM_TIMES_TO_TRACK);
-
- // when
- Time.advanceTimeSecs(secondsToAdvance);
- int seconds = tracker.secondsSinceOldestModification();
-
- // then
- assertThat(seconds).isEqualTo(secondsToAdvance);
- }
- }
-
- @DataProvider
- public Object[][] simulatedTrackerIterations() {
- return new Object[][]{ { 1, new int[]{ 0, 1 }, new int[]{ 0, 0 } }, { 1, new int[]{ 0, 2 }, new int[]{ 0, 0 } },
- { 2, new int[]{ 2, 2 }, new int[]{ 2, 2 } }, { 2, new int[]{ 0, 4 }, new int[]{ 0, 4 } },
- { 1, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 0, 0, 0, 0, 0, 0, 0 } },
- { 1, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 0, 0, 0, 0, 0, 0, 0 } },
- { 2, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 1, 1, 1, 1, 1, 1 } },
- { 2, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 2, 2, 2, 2, 2, 2 } },
- { 2, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 2, 3, 4, 5, 6, 7 } },
- { 3, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 2, 2, 2, 2, 2 } },
- { 3, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 5, 7, 9, 11, 13 } },
- { 3, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 4, 4, 4, 4, 4 } },
- { 4, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 3, 3, 3, 3, 3 } },
- { 4, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 6, 9, 12, 15, 18 } },
- { 4, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 6, 6, 6, 6, 6 } },
- { 5, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 3, 4, 4, 4, 4 } },
- { 5, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 6, 10, 14, 18, 22 } },
- { 5, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 6, 8, 8, 8, 8 } },
- { 6, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 3, 4, 5, 5, 5 } },
- { 6, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 6, 10, 15, 20, 25 } },
- { 6, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 6, 8, 10, 10, 10 } },
- { 3, new int[]{ 1, 2, 3 }, new int[]{ 1, 3, 5 } } };
- }
+ private static final int ANY_NUM_TIMES_TO_TRACK = 3;
+ private static final int MILLIS_IN_SEC = 1000;
+
+ @DataProvider
+ public Object[][] illegalNumTimesData() {
+ return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalNumTimesData")
+ public void negativeOrZeroNumTimesToTrackShouldThrowIAE(int numTimesToTrack) {
+ new NthLastModifiedTimeTracker(numTimesToTrack);
+ }
+
+ @DataProvider
+ public Object[][] legalNumTimesData() {
+ return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
+ }
+
+ @Test(dataProvider = "legalNumTimesData")
+ public void positiveNumTimesToTrackShouldBeOk(int numTimesToTrack) {
+ new NthLastModifiedTimeTracker(numTimesToTrack);
+ }
+
+ @DataProvider
+ public Object[][] whenNotYetMarkedAsModifiedData() {
+ return new Object[][]{ { 0 }, { 1 }, { 2 }, { 3 }, { 4 }, { 5 }, { 8 }, { 10 } };
+ }
+
+ @Test(dataProvider = "whenNotYetMarkedAsModifiedData")
+ public void shouldReturnCorrectModifiedTimeEvenWhenNotYetMarkedAsModified(int secondsToAdvance) {
+ // given
+ try (SimulatedTime t = new SimulatedTime()) {
+ NthLastModifiedTimeTracker tracker = new NthLastModifiedTimeTracker(ANY_NUM_TIMES_TO_TRACK);
+
+ // when
+ Time.advanceTimeSecs(secondsToAdvance);
+ int seconds = tracker.secondsSinceOldestModification();
+
+ // then
+ assertThat(seconds).isEqualTo(secondsToAdvance);
+ }
+ }
+
+ @DataProvider
+ public Object[][] simulatedTrackerIterations() {
+ return new Object[][]{
+ { 1, new int[]{ 0, 1 }, new int[]{ 0, 0 } }, { 1, new int[]{ 0, 2 }, new int[]{ 0, 0 } },
+ { 2, new int[]{ 2, 2 }, new int[]{ 2, 2 } }, { 2, new int[]{ 0, 4 }, new int[]{ 0, 4 } },
+ { 1, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 0, 0, 0, 0, 0, 0, 0 } },
+ { 1, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 0, 0, 0, 0, 0, 0, 0 } },
+ { 2, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 1, 1, 1, 1, 1, 1 } },
+ { 2, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 2, 2, 2, 2, 2, 2 } },
+ { 2, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 2, 3, 4, 5, 6, 7 } },
+ { 3, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 2, 2, 2, 2, 2 } },
+ { 3, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 5, 7, 9, 11, 13 } },
+ { 3, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 4, 4, 4, 4, 4 } },
+ { 4, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 3, 3, 3, 3, 3 } },
+ { 4, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 6, 9, 12, 15, 18 } },
+ { 4, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 6, 6, 6, 6, 6 } },
+ { 5, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 3, 4, 4, 4, 4 } },
+ { 5, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 6, 10, 14, 18, 22 } },
+ { 5, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 6, 8, 8, 8, 8 } },
+ { 6, new int[]{ 1, 1, 1, 1, 1, 1, 1 }, new int[]{ 1, 2, 3, 4, 5, 5, 5 } },
+ { 6, new int[]{ 1, 2, 3, 4, 5, 6, 7 }, new int[]{ 1, 3, 6, 10, 15, 20, 25 } },
+ { 6, new int[]{ 2, 2, 2, 2, 2, 2, 2 }, new int[]{ 2, 4, 6, 8, 10, 10, 10 } },
+ { 3, new int[]{ 1, 2, 3 }, new int[]{ 1, 3, 5 } }
+ };
+ }
@Test(dataProvider = "simulatedTrackerIterations")
public void shouldReturnCorrectModifiedTimeWhenMarkedAsModified(int numTimesToTrack,
- int[] secondsToAdvancePerIteration, int[] expLastModifiedTimes) {
+ int[] secondsToAdvancePerIteration, int[] expLastModifiedTimes) {
// given
try (SimulatedTime t = new SimulatedTime()) {
NthLastModifiedTimeTracker tracker = new NthLastModifiedTimeTracker(numTimesToTrack);
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java
index 9837569..aa4d470 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankableObjectWithFieldsTest.java
@@ -1,252 +1,275 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.tools;
-import org.apache.storm.tuple.Tuple;
import com.google.common.collect.Lists;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
import java.util.ArrayList;
import java.util.List;
+import org.apache.storm.tuple.Tuple;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
import static org.fest.assertions.api.Assertions.assertThat;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
public class RankableObjectWithFieldsTest {
- private static final Object ANY_OBJECT = new Object();
- private static final long ANY_COUNT = 271;
- private static final String ANY_FIELD = "someAdditionalField";
- private static final int GREATER_THAN = 1;
- private static final int EQUAL_TO = 0;
- private static final int SMALLER_THAN = -1;
-
- @Test(expectedExceptions = IllegalArgumentException.class)
- public void constructorWithNullObjectAndNoFieldsShouldThrowIAE() {
- new RankableObjectWithFields(null, ANY_COUNT);
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class)
- public void constructorWithNullObjectAndFieldsShouldThrowIAE() {
- Object someAdditionalField = new Object();
- new RankableObjectWithFields(null, ANY_COUNT, someAdditionalField);
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class)
- public void constructorWithNegativeCountAndNoFieldsShouldThrowIAE() {
- new RankableObjectWithFields(ANY_OBJECT, -1);
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class)
- public void constructorWithNegativeCountAndFieldsShouldThrowIAE() {
- Object someAdditionalField = new Object();
- new RankableObjectWithFields(ANY_OBJECT, -1, someAdditionalField);
- }
-
- @Test
- public void shouldBeEqualToItself() {
- RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT);
- assertThat(r).isEqualTo(r);
- }
-
- @DataProvider
- public Object[][] otherClassesData() {
- return new Object[][]{ { new String("foo") }, { new Object() }, { Integer.valueOf(4) }, { Lists.newArrayList(7, 8,
- 9) } };
- }
-
- @Test(dataProvider = "otherClassesData")
- public void shouldNotBeEqualToInstancesOfOtherClasses(Object notARankable) {
- RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT);
- assertFalse(r.equals(notARankable), r + " is equal to " + notARankable + " but it should not be");
- }
-
- @DataProvider
- public Object[][] falseDuplicatesData() {
- return new Object[][]{ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1) },
- { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("Foo", 1) },
- { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("FOO", 1) },
- { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 1) },
- { new RankableObjectWithFields("", 0), new RankableObjectWithFields("", 1) }, { new RankableObjectWithFields("",
- 1), new RankableObjectWithFields("bar", 1) } };
- }
-
- @Test(dataProvider = "falseDuplicatesData")
- public void shouldNotBeEqualToFalseDuplicates(RankableObjectWithFields r, RankableObjectWithFields falseDuplicate) {
- assertFalse(r.equals(falseDuplicate), r + " is equal to " + falseDuplicate + " but it should not be");
- }
-
- @Test(dataProvider = "falseDuplicatesData")
- public void shouldHaveDifferentHashCodeThanFalseDuplicates(RankableObjectWithFields r,
- RankableObjectWithFields falseDuplicate) {
- assertThat(r.hashCode()).isNotEqualTo(falseDuplicate.hashCode());
- }
-
- @DataProvider
- public Object[][] trueDuplicatesData() {
- return new Object[][]{ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0) },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0, "someOtherField") },
- { new RankableObjectWithFields("foo", 0, "someField"), new RankableObjectWithFields("foo", 0,
- "someOtherField") } };
- }
-
- @Test(dataProvider = "trueDuplicatesData")
- public void shouldBeEqualToTrueDuplicates(RankableObjectWithFields r, RankableObjectWithFields trueDuplicate) {
- assertTrue(r.equals(trueDuplicate), r + " is not equal to " + trueDuplicate + " but it should be");
- }
-
- @Test(dataProvider = "trueDuplicatesData")
- public void shouldHaveSameHashCodeAsTrueDuplicates(RankableObjectWithFields r,
- RankableObjectWithFields trueDuplicate) {
- assertThat(r.hashCode()).isEqualTo(trueDuplicate.hashCode());
- }
-
- @DataProvider
- public Object[][] compareToData() {
- return new Object[][]{ { new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("foo", 0),
- GREATER_THAN }, { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("foo", 0),
- GREATER_THAN }, { new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("bar", 0),
- GREATER_THAN }, { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 0),
- GREATER_THAN }, { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0), EQUAL_TO },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 0), EQUAL_TO },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1000), SMALLER_THAN },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1), SMALLER_THAN },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1), SMALLER_THAN },
- { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1000), SMALLER_THAN }, };
- }
-
- @Test(dataProvider = "compareToData")
- public void verifyCompareTo(RankableObjectWithFields first, RankableObjectWithFields second, int expCompareToValue) {
- assertThat(first.compareTo(second)).isEqualTo(expCompareToValue);
- }
-
- @DataProvider
- public Object[][] toStringData() {
- return new Object[][]{ { new String("foo"), 0L }, { new String("BAR"), 8L } };
- }
-
- @Test(dataProvider = "toStringData")
- public void toStringShouldContainStringRepresentationsOfObjectAndCount(Object obj, long count) {
- // given
- RankableObjectWithFields r = new RankableObjectWithFields(obj, count);
-
- // when
- String strRepresentation = r.toString();
-
- // then
- assertThat(strRepresentation).contains(obj.toString()).contains("" + count);
- }
-
- @Test
- public void shouldReturnTheObject() {
- // given
- RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD);
-
- // when
- Object obj = r.getObject();
-
- // then
- assertThat(obj).isEqualTo(ANY_OBJECT);
- }
-
- @Test
- public void shouldReturnTheCount() {
- // given
- RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD);
-
- // when
- long count = r.getCount();
-
- // then
- assertThat(count).isEqualTo(ANY_COUNT);
- }
-
- @DataProvider
- public Object[][] fieldsData() {
- return new Object[][]{ { ANY_OBJECT, ANY_COUNT, new Object[]{ ANY_FIELD } },
- { "quux", 42L, new Object[]{ "one", "two", "three" } } };
- }
-
- @Test(dataProvider = "fieldsData")
- public void shouldReturnTheFields(Object obj, long count, Object[] fields) {
- // given
- RankableObjectWithFields r = new RankableObjectWithFields(obj, count, fields);
-
- // when
- List<Object> actualFields = r.getFields();
-
- // then
- assertThat(actualFields).isEqualTo(Lists.newArrayList(fields));
- }
-
- @Test(expectedExceptions = UnsupportedOperationException.class)
- public void fieldsShouldBeImmutable() {
- // given
- RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD);
-
- // when
- List<Object> fields = r.getFields();
- // try to modify the list, which should fail
- fields.remove(0);
-
- // then (exception)
- }
-
- @Test
- public void shouldCreateRankableObjectFromTuple() {
- // given
- Tuple tuple = mock(Tuple.class);
- List<Object> tupleValues = Lists.newArrayList(ANY_OBJECT, ANY_COUNT, ANY_FIELD);
- when(tuple.getValues()).thenReturn(tupleValues);
-
- // when
- RankableObjectWithFields r = RankableObjectWithFields.from(tuple);
-
- // then
- assertThat(r.getObject()).isEqualTo(ANY_OBJECT);
- assertThat(r.getCount()).isEqualTo(ANY_COUNT);
- List<Object> fields = new ArrayList<Object>();
- fields.add(ANY_FIELD);
- assertThat(r.getFields()).isEqualTo(fields);
-
- }
-
- @DataProvider
- public Object[][] copyData() {
- return new Object[][]{ { new RankableObjectWithFields("foo", 0) }, { new RankableObjectWithFields("foo", 3,
- "someOtherField") }, { new RankableObjectWithFields("foo", 0, "someField") } };
- }
-
- // TODO: What would be a good test to ensure that RankableObjectWithFields is at least somewhat defensively copied?
- // The contract of Rankable#copy() returns a Rankable value, not a RankableObjectWithFields.
- @Test(dataProvider = "copyData")
- public void copyShouldReturnCopy(RankableObjectWithFields original) {
- // given
-
- // when
- Rankable copy = original.copy();
-
- // then
- assertThat(copy.getObject()).isEqualTo(original.getObject());
- assertThat(copy.getCount()).isEqualTo(original.getCount());
- }
+ private static final Object ANY_OBJECT = new Object();
+ private static final long ANY_COUNT = 271;
+ private static final String ANY_FIELD = "someAdditionalField";
+ private static final int GREATER_THAN = 1;
+ private static final int EQUAL_TO = 0;
+ private static final int SMALLER_THAN = -1;
+
+ @Test(expectedExceptions = IllegalArgumentException.class)
+ public void constructorWithNullObjectAndNoFieldsShouldThrowIAE() {
+ new RankableObjectWithFields(null, ANY_COUNT);
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class)
+ public void constructorWithNullObjectAndFieldsShouldThrowIAE() {
+ Object someAdditionalField = new Object();
+ new RankableObjectWithFields(null, ANY_COUNT, someAdditionalField);
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class)
+ public void constructorWithNegativeCountAndNoFieldsShouldThrowIAE() {
+ new RankableObjectWithFields(ANY_OBJECT, -1);
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class)
+ public void constructorWithNegativeCountAndFieldsShouldThrowIAE() {
+ Object someAdditionalField = new Object();
+ new RankableObjectWithFields(ANY_OBJECT, -1, someAdditionalField);
+ }
+
+ @Test
+ public void shouldBeEqualToItself() {
+ RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT);
+ assertThat(r).isEqualTo(r);
+ }
+
+ @DataProvider
+ public Object[][] otherClassesData() {
+ return new Object[][]{
+ { new String("foo") }, { new Object() }, { Integer.valueOf(4) }, {
+ Lists.newArrayList(7, 8,
+ 9)
+ }
+ };
+ }
+
+ @Test(dataProvider = "otherClassesData")
+ public void shouldNotBeEqualToInstancesOfOtherClasses(Object notARankable) {
+ RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT);
+ assertFalse(r.equals(notARankable), r + " is equal to " + notARankable + " but it should not be");
+ }
+
+ @DataProvider
+ public Object[][] falseDuplicatesData() {
+ return new Object[][]{
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1) },
+ { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("Foo", 1) },
+ { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("FOO", 1) },
+ { new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 1) },
+ { new RankableObjectWithFields("", 0), new RankableObjectWithFields("", 1) }, {
+ new RankableObjectWithFields("",
+ 1), new RankableObjectWithFields("bar", 1)
+ }
+ };
+ }
+
+ @Test(dataProvider = "falseDuplicatesData")
+ public void shouldNotBeEqualToFalseDuplicates(RankableObjectWithFields r, RankableObjectWithFields falseDuplicate) {
+ assertFalse(r.equals(falseDuplicate), r + " is equal to " + falseDuplicate + " but it should not be");
+ }
+
+ @Test(dataProvider = "falseDuplicatesData")
+ public void shouldHaveDifferentHashCodeThanFalseDuplicates(RankableObjectWithFields r,
+ RankableObjectWithFields falseDuplicate) {
+ assertThat(r.hashCode()).isNotEqualTo(falseDuplicate.hashCode());
+ }
+
+ @DataProvider
+ public Object[][] trueDuplicatesData() {
+ return new Object[][]{
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0) },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0, "someOtherField") },
+ {
+ new RankableObjectWithFields("foo", 0, "someField"), new RankableObjectWithFields("foo", 0,
+ "someOtherField")
+ }
+ };
+ }
+
+ @Test(dataProvider = "trueDuplicatesData")
+ public void shouldBeEqualToTrueDuplicates(RankableObjectWithFields r, RankableObjectWithFields trueDuplicate) {
+ assertTrue(r.equals(trueDuplicate), r + " is not equal to " + trueDuplicate + " but it should be");
+ }
+
+ @Test(dataProvider = "trueDuplicatesData")
+ public void shouldHaveSameHashCodeAsTrueDuplicates(RankableObjectWithFields r,
+ RankableObjectWithFields trueDuplicate) {
+ assertThat(r.hashCode()).isEqualTo(trueDuplicate.hashCode());
+ }
+
+ @DataProvider
+ public Object[][] compareToData() {
+ return new Object[][]{
+ {
+ new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("foo", 0),
+ GREATER_THAN
+ }, {
+ new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("foo", 0),
+ GREATER_THAN
+ }, {
+ new RankableObjectWithFields("foo", 1000), new RankableObjectWithFields("bar", 0),
+ GREATER_THAN
+ }, {
+ new RankableObjectWithFields("foo", 1), new RankableObjectWithFields("bar", 0),
+ GREATER_THAN
+ }, { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 0), EQUAL_TO },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 0), EQUAL_TO },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1000), SMALLER_THAN },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("foo", 1), SMALLER_THAN },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1), SMALLER_THAN },
+ { new RankableObjectWithFields("foo", 0), new RankableObjectWithFields("bar", 1000), SMALLER_THAN },
+ };
+ }
+
+ @Test(dataProvider = "compareToData")
+ public void verifyCompareTo(RankableObjectWithFields first, RankableObjectWithFields second, int expCompareToValue) {
+ assertThat(first.compareTo(second)).isEqualTo(expCompareToValue);
+ }
+
+ @DataProvider
+ public Object[][] toStringData() {
+ return new Object[][]{ { new String("foo"), 0L }, { new String("BAR"), 8L } };
+ }
+
+ @Test(dataProvider = "toStringData")
+ public void toStringShouldContainStringRepresentationsOfObjectAndCount(Object obj, long count) {
+ // given
+ RankableObjectWithFields r = new RankableObjectWithFields(obj, count);
+
+ // when
+ String strRepresentation = r.toString();
+
+ // then
+ assertThat(strRepresentation).contains(obj.toString()).contains("" + count);
+ }
+
+ @Test
+ public void shouldReturnTheObject() {
+ // given
+ RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD);
+
+ // when
+ Object obj = r.getObject();
+
+ // then
+ assertThat(obj).isEqualTo(ANY_OBJECT);
+ }
+
+ @Test
+ public void shouldReturnTheCount() {
+ // given
+ RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD);
+
+ // when
+ long count = r.getCount();
+
+ // then
+ assertThat(count).isEqualTo(ANY_COUNT);
+ }
+
+ @DataProvider
+ public Object[][] fieldsData() {
+ return new Object[][]{
+ { ANY_OBJECT, ANY_COUNT, new Object[]{ ANY_FIELD } },
+ { "quux", 42L, new Object[]{ "one", "two", "three" } }
+ };
+ }
+
+ @Test(dataProvider = "fieldsData")
+ public void shouldReturnTheFields(Object obj, long count, Object[] fields) {
+ // given
+ RankableObjectWithFields r = new RankableObjectWithFields(obj, count, fields);
+
+ // when
+ List<Object> actualFields = r.getFields();
+
+ // then
+ assertThat(actualFields).isEqualTo(Lists.newArrayList(fields));
+ }
+
+ @Test(expectedExceptions = UnsupportedOperationException.class)
+ public void fieldsShouldBeImmutable() {
+ // given
+ RankableObjectWithFields r = new RankableObjectWithFields(ANY_OBJECT, ANY_COUNT, ANY_FIELD);
+
+ // when
+ List<Object> fields = r.getFields();
+ // try to modify the list, which should fail
+ fields.remove(0);
+
+ // then (exception)
+ }
+
+ @Test
+ public void shouldCreateRankableObjectFromTuple() {
+ // given
+ Tuple tuple = mock(Tuple.class);
+ List<Object> tupleValues = Lists.newArrayList(ANY_OBJECT, ANY_COUNT, ANY_FIELD);
+ when(tuple.getValues()).thenReturn(tupleValues);
+
+ // when
+ RankableObjectWithFields r = RankableObjectWithFields.from(tuple);
+
+ // then
+ assertThat(r.getObject()).isEqualTo(ANY_OBJECT);
+ assertThat(r.getCount()).isEqualTo(ANY_COUNT);
+ List<Object> fields = new ArrayList<Object>();
+ fields.add(ANY_FIELD);
+ assertThat(r.getFields()).isEqualTo(fields);
+
+ }
+
+ @DataProvider
+ public Object[][] copyData() {
+ return new Object[][]{
+ { new RankableObjectWithFields("foo", 0) }, {
+ new RankableObjectWithFields("foo", 3,
+ "someOtherField")
+ }, { new RankableObjectWithFields("foo", 0, "someField") }
+ };
+ }
+
+ // TODO: What would be a good test to ensure that RankableObjectWithFields is at least somewhat defensively copied?
+ // The contract of Rankable#copy() returns a Rankable value, not a RankableObjectWithFields.
+ @Test(dataProvider = "copyData")
+ public void copyShouldReturnCopy(RankableObjectWithFields original) {
+ // given
+
+ // when
+ Rankable copy = original.copy();
+
+ // then
+ assertThat(copy.getObject()).isEqualTo(original.getObject());
+ assertThat(copy.getCount()).isEqualTo(original.getCount());
+ }
}
[04/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreTest.java b/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreTest.java
index 45b6468..57b0750 100644
--- a/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreTest.java
+++ b/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreTest.java
@@ -1,27 +1,22 @@
-
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.blobstore;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
+import java.nio.file.Files;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -29,9 +24,7 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
-
import javax.security.auth.Subject;
-
import org.apache.commons.io.FileUtils;
import org.apache.storm.Config;
import org.apache.storm.generated.AuthorizationException;
@@ -53,423 +46,422 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.spy;
-import java.nio.file.Files;
-
public class BlobStoreTest {
- private static final Logger LOG = LoggerFactory.getLogger(BlobStoreTest.class);
- URI base;
- File baseFile;
- private static Map<String, Object> conf = new HashMap();
- public static final int READ = 0x01;
- public static final int WRITE = 0x02;
- public static final int ADMIN = 0x04;
-
- @Before
- public void init() {
- initializeConfigs();
- baseFile = new File("target/blob-store-test-"+UUID.randomUUID());
- base = baseFile.toURI();
- }
-
- @After
- public void cleanup() throws IOException {
- FileUtils.deleteDirectory(baseFile);
- }
-
- // Method which initializes nimbus admin
- public static void initializeConfigs() {
- conf.put(Config.NIMBUS_ADMINS,"admin");
- conf.put(Config.NIMBUS_SUPERVISOR_USERS,"supervisor");
- }
-
- // Gets Nimbus Subject with NimbusPrincipal set on it
- public static Subject getNimbusSubject() {
- Subject nimbus = new Subject();
- nimbus.getPrincipals().add(new NimbusPrincipal());
- return nimbus;
- }
-
- // Overloading the assertStoreHasExactly method accomodate Subject in order to check for authorization
- public static void assertStoreHasExactly(BlobStore store, Subject who, String ... keys)
- throws IOException, KeyNotFoundException, AuthorizationException {
- Set<String> expected = new HashSet<String>(Arrays.asList(keys));
- Set<String> found = new HashSet<String>();
- Iterator<String> c = store.listKeys();
- while (c.hasNext()) {
- String keyName = c.next();
- found.add(keyName);
- }
- Set<String> extra = new HashSet<String>(found);
- extra.removeAll(expected);
- assertTrue("Found extra keys in the blob store "+extra, extra.isEmpty());
- Set<String> missing = new HashSet<String>(expected);
- missing.removeAll(found);
- assertTrue("Found keys missing from the blob store "+missing, missing.isEmpty());
- }
-
- public static void assertStoreHasExactly(BlobStore store, String ... keys)
- throws IOException, KeyNotFoundException, AuthorizationException {
- assertStoreHasExactly(store, null, keys);
- }
-
- // Overloading the readInt method accomodate Subject in order to check for authorization (security turned on)
- public static int readInt(BlobStore store, Subject who, String key)
- throws IOException, KeyNotFoundException, AuthorizationException {
- InputStream in = store.getBlob(key, who);
- try {
- return in.read();
- } finally {
- in.close();
+ public static final int READ = 0x01;
+ public static final int WRITE = 0x02;
+ public static final int ADMIN = 0x04;
+ private static final Logger LOG = LoggerFactory.getLogger(BlobStoreTest.class);
+ private static Map<String, Object> conf = new HashMap();
+ URI base;
+ File baseFile;
+
+ // Method which initializes nimbus admin
+ public static void initializeConfigs() {
+ conf.put(Config.NIMBUS_ADMINS, "admin");
+ conf.put(Config.NIMBUS_SUPERVISOR_USERS, "supervisor");
}
- }
-
- public static int readInt(BlobStore store, String key)
- throws IOException, KeyNotFoundException, AuthorizationException {
- return readInt(store, null, key);
- }
-
- public static void readAssertEquals(BlobStore store, String key, int value)
- throws IOException, KeyNotFoundException, AuthorizationException {
- assertEquals(value, readInt(store, key));
- }
-
- // Checks for assertion when we turn on security
- public void readAssertEqualsWithAuth(BlobStore store, Subject who, String key, int value)
- throws IOException, KeyNotFoundException, AuthorizationException {
- assertEquals(value, readInt(store, who, key));
- }
-
- private LocalFsBlobStore initLocalFs() {
- LocalFsBlobStore store = new LocalFsBlobStore();
- // Spy object that tries to mock the real object store
- LocalFsBlobStore spy = spy(store);
- Mockito.doNothing().when(spy).checkForBlobUpdate("test");
- Mockito.doNothing().when(spy).checkForBlobUpdate("other");
- Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-WE");
- Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-DEF");
- Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-acls");
- Map<String, Object> conf = Utils.readStormConfig();
- conf.put(Config.STORM_LOCAL_DIR, baseFile.getAbsolutePath());
- conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,"org.apache.storm.security.auth.DefaultPrincipalToLocal");
- spy.prepare(conf, null, null);
- return spy;
- }
-
- @Test
- public void testLocalFsWithAuth() throws Exception {
- testWithAuthentication(initLocalFs());
- }
-
- @Test
- public void testBasicLocalFs() throws Exception {
- testBasic(initLocalFs());
- }
-
- @Test
- public void testMultipleLocalFs() throws Exception {
- testMultiple(initLocalFs());
- }
-
- @Test
- public void testDeleteAfterFailedCreate() throws Exception{
- //Check that a blob can be deleted when a temporary file exists in the blob directory
- LocalFsBlobStore store = initLocalFs();
-
- String key = "test";
- SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler
- .WORLD_EVERYTHING);
- try (AtomicOutputStream out = store.createBlob(key, metadata, null)) {
- out.write(1);
- File blobDir = store.getKeyDataDir(key);
- Files.createFile(blobDir.toPath().resolve("tempFile.tmp"));
- }
-
- store.deleteBlob("test",null);
-
- }
-
- public Subject getSubject(String name) {
- Subject subject = new Subject();
- SingleUserPrincipal user = new SingleUserPrincipal(name);
- subject.getPrincipals().add(user);
- return subject;
- }
-
- // Check for Blobstore with authentication
- public void testWithAuthentication(BlobStore store) throws Exception {
- //Test for Nimbus Admin
- Subject admin = getSubject("admin");
- assertStoreHasExactly(store);
- SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- try (AtomicOutputStream out = store.createBlob("test", metadata, admin)) {
- assertStoreHasExactly(store, "test");
- out.write(1);
- }
- store.deleteBlob("test", admin);
-
- //Test for Supervisor Admin
- Subject supervisor = getSubject("supervisor");
- assertStoreHasExactly(store);
- metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- try (AtomicOutputStream out = store.createBlob("test", metadata, supervisor)) {
- assertStoreHasExactly(store, "test");
- out.write(1);
- }
- store.deleteBlob("test", supervisor);
-
- //Test for Nimbus itself as a user
- Subject nimbus = getNimbusSubject();
- assertStoreHasExactly(store);
- metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- try (AtomicOutputStream out = store.createBlob("test", metadata, nimbus)) {
- assertStoreHasExactly(store, "test");
- out.write(1);
- }
- store.deleteBlob("test", nimbus);
- // Test with a dummy test_subject for cases where subject !=null (security turned on)
- Subject who = getSubject("test_subject");
- assertStoreHasExactly(store);
-
- // Tests for case when subject != null (security turned on) and
- // acls for the blob are set to WORLD_EVERYTHING
- metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
- try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
- out.write(1);
- }
- assertStoreHasExactly(store, "test");
- // Testing whether acls are set to WORLD_EVERYTHING
- assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
- readAssertEqualsWithAuth(store, who, "test", 1);
-
- LOG.info("Deleting test");
- store.deleteBlob("test", who);
- assertStoreHasExactly(store);
-
- // Tests for case when subject != null (security turned on) and
- // acls are not set for the blob (DEFAULT)
- LOG.info("Creating test again");
- metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
- out.write(2);
+ // Gets Nimbus Subject with NimbusPrincipal set on it
+ public static Subject getNimbusSubject() {
+ Subject nimbus = new Subject();
+ nimbus.getPrincipals().add(new NimbusPrincipal());
+ return nimbus;
}
- assertStoreHasExactly(store, "test");
- // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain WORLD_EVERYTHING because
- // the subject is neither null nor empty. The ACL should however contain USER_EVERYTHING as user needs to have
- // complete access to the blob
- assertTrue("ACL does not contain WORLD_EVERYTHING", !metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
- readAssertEqualsWithAuth(store, who, "test", 2);
-
- LOG.info("Updating test");
- try (AtomicOutputStream out = store.updateBlob("test", who)) {
- out.write(3);
+
+ // Overloading the assertStoreHasExactly method accomodate Subject in order to check for authorization
+ public static void assertStoreHasExactly(BlobStore store, Subject who, String... keys)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ Set<String> expected = new HashSet<String>(Arrays.asList(keys));
+ Set<String> found = new HashSet<String>();
+ Iterator<String> c = store.listKeys();
+ while (c.hasNext()) {
+ String keyName = c.next();
+ found.add(keyName);
+ }
+ Set<String> extra = new HashSet<String>(found);
+ extra.removeAll(expected);
+ assertTrue("Found extra keys in the blob store " + extra, extra.isEmpty());
+ Set<String> missing = new HashSet<String>(expected);
+ missing.removeAll(found);
+ assertTrue("Found keys missing from the blob store " + missing, missing.isEmpty());
}
- assertStoreHasExactly(store, "test");
- readAssertEqualsWithAuth(store, who, "test", 3);
-
- LOG.info("Updating test again");
- try (AtomicOutputStream out = store.updateBlob("test", who)) {
- out.write(4);
- out.flush();
- LOG.info("SLEEPING");
- Thread.sleep(2);
- assertStoreHasExactly(store, "test");
- readAssertEqualsWithAuth(store, who, "test", 3);
+
+ public static void assertStoreHasExactly(BlobStore store, String... keys)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ assertStoreHasExactly(store, null, keys);
}
- // Test for subject with no principals and acls set to WORLD_EVERYTHING
- who = new Subject();
- metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
- LOG.info("Creating test");
- try (AtomicOutputStream out = store.createBlob("test-empty-subject-WE", metadata, who)) {
- out.write(2);
+ // Overloading the readInt method accomodate Subject in order to check for authorization (security turned on)
+ public static int readInt(BlobStore store, Subject who, String key)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ InputStream in = store.getBlob(key, who);
+ try {
+ return in.read();
+ } finally {
+ in.close();
+ }
}
- assertStoreHasExactly(store, "test-empty-subject-WE", "test");
- // Testing whether acls are set to WORLD_EVERYTHING
- assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
- readAssertEqualsWithAuth(store, who, "test-empty-subject-WE", 2);
-
- // Test for subject with no principals and acls set to DEFAULT
- who = new Subject();
- metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- LOG.info("Creating other");
-
- try (AtomicOutputStream out = store.createBlob("test-empty-subject-DEF", metadata, who)) {
- out.write(2);
+
+ public static int readInt(BlobStore store, String key)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ return readInt(store, null, key);
}
- assertStoreHasExactly(store, "test-empty-subject-DEF", "test", "test-empty-subject-WE");
- // Testing whether acls are set to WORLD_EVERYTHING
- assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
- readAssertEqualsWithAuth(store, who, "test-empty-subject-DEF", 2);
-
- if (store instanceof LocalFsBlobStore) {
- ((LocalFsBlobStore) store).fullCleanup(1);
- } else {
- fail("Error the blobstore is of unknowntype");
+
+ public static void readAssertEquals(BlobStore store, String key, int value)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ assertEquals(value, readInt(store, key));
}
- }
-
- public void testBasic(BlobStore store) throws Exception {
- assertStoreHasExactly(store);
- LOG.info("Creating test");
- // Tests for case when subject == null (security turned off) and
- // acls for the blob are set to WORLD_EVERYTHING
- SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler
- .WORLD_EVERYTHING);
- try (AtomicOutputStream out = store.createBlob("test", metadata, null)) {
- out.write(1);
+
+ @Before
+ public void init() {
+ initializeConfigs();
+ baseFile = new File("target/blob-store-test-" + UUID.randomUUID());
+ base = baseFile.toURI();
}
- assertStoreHasExactly(store, "test");
- // Testing whether acls are set to WORLD_EVERYTHING
- assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
- readAssertEquals(store, "test", 1);
-
- LOG.info("Deleting test");
- store.deleteBlob("test", null);
- assertStoreHasExactly(store);
-
- // The following tests are run for both hdfs and local store to test the
- // update blob interface
- metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
- LOG.info("Creating test again");
- try (AtomicOutputStream out = store.createBlob("test", metadata, null)) {
- out.write(2);
+
+ @After
+ public void cleanup() throws IOException {
+ FileUtils.deleteDirectory(baseFile);
}
- assertStoreHasExactly(store, "test");
- if (store instanceof LocalFsBlobStore) {
- assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+
+ // Checks for assertion when we turn on security
+ public void readAssertEqualsWithAuth(BlobStore store, Subject who, String key, int value)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ assertEquals(value, readInt(store, who, key));
}
- readAssertEquals(store, "test", 2);
- LOG.info("Updating test");
- try (AtomicOutputStream out = store.updateBlob("test", null)) {
- out.write(3);
+
+ private LocalFsBlobStore initLocalFs() {
+ LocalFsBlobStore store = new LocalFsBlobStore();
+ // Spy object that tries to mock the real object store
+ LocalFsBlobStore spy = spy(store);
+ Mockito.doNothing().when(spy).checkForBlobUpdate("test");
+ Mockito.doNothing().when(spy).checkForBlobUpdate("other");
+ Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-WE");
+ Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-DEF");
+ Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-acls");
+ Map<String, Object> conf = Utils.readStormConfig();
+ conf.put(Config.STORM_LOCAL_DIR, baseFile.getAbsolutePath());
+ conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal");
+ spy.prepare(conf, null, null);
+ return spy;
}
- assertStoreHasExactly(store, "test");
- readAssertEquals(store, "test", 3);
-
- LOG.info("Updating test again");
- try (AtomicOutputStream out = store.updateBlob("test", null)) {
- out.write(4);
- out.flush();
- LOG.info("SLEEPING");
- Thread.sleep(2);
+
+ @Test
+ public void testLocalFsWithAuth() throws Exception {
+ testWithAuthentication(initLocalFs());
}
- // Tests for case when subject == null (security turned off) and
- // acls for the blob are set to DEFAULT (Empty ACL List) only for LocalFsBlobstore
- if (store instanceof LocalFsBlobStore) {
- metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- LOG.info("Creating test for empty acls when security is off");
- try (AtomicOutputStream out = store.createBlob("test-empty-acls", metadata, null)) {
- LOG.info("metadata {}", metadata);
- out.write(2);
- }
- assertStoreHasExactly(store, "test-empty-acls", "test");
- // Testing whether acls are set to WORLD_EVERYTHING, Here we are testing only for LocalFsBlobstore
- // as the HdfsBlobstore gets the subject information of the local system user and behaves as it is
- // always authenticated.
- assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.get_acl().toString().contains("OTHER"));
-
- LOG.info("Deleting test-empty-acls");
- store.deleteBlob("test-empty-acls", null);
+ @Test
+ public void testBasicLocalFs() throws Exception {
+ testBasic(initLocalFs());
}
- if (store instanceof LocalFsBlobStore) {
- ((LocalFsBlobStore) store).fullCleanup(1);
- } else {
- fail("Error the blobstore is of unknowntype");
+ @Test
+ public void testMultipleLocalFs() throws Exception {
+ testMultiple(initLocalFs());
}
- }
+ @Test
+ public void testDeleteAfterFailedCreate() throws Exception {
+ //Check that a blob can be deleted when a temporary file exists in the blob directory
+ LocalFsBlobStore store = initLocalFs();
- public void testMultiple(BlobStore store) throws Exception {
+ String key = "test";
+ SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler
+ .WORLD_EVERYTHING);
+ try (AtomicOutputStream out = store.createBlob(key, metadata, null)) {
+ out.write(1);
+ File blobDir = store.getKeyDataDir(key);
+ Files.createFile(blobDir.toPath().resolve("tempFile.tmp"));
+ }
- assertStoreHasExactly(store);
- LOG.info("Creating test");
- try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler
- .WORLD_EVERYTHING), null)) {
- out.write(1);
- }
- assertStoreHasExactly(store, "test");
- readAssertEquals(store, "test", 1);
+ store.deleteBlob("test", null);
- LOG.info("Creating other");
- try (AtomicOutputStream out = store.createBlob("other", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
- null)) {
- out.write(2);
}
- assertStoreHasExactly(store, "test", "other");
- readAssertEquals(store, "test", 1);
- readAssertEquals(store, "other", 2);
- LOG.info("Updating other");
- try (AtomicOutputStream out = store.updateBlob("other", null)) {
- out.write(5);
+ public Subject getSubject(String name) {
+ Subject subject = new Subject();
+ SingleUserPrincipal user = new SingleUserPrincipal(name);
+ subject.getPrincipals().add(user);
+ return subject;
}
- assertStoreHasExactly(store, "test", "other");
- readAssertEquals(store, "test", 1);
- readAssertEquals(store, "other", 5);
-
- LOG.info("Deleting test");
- store.deleteBlob("test", null);
- assertStoreHasExactly(store, "other");
- readAssertEquals(store, "other", 5);
-
- LOG.info("Creating test again");
- try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
- null)) {
- out.write(2);
- }
- assertStoreHasExactly(store, "test", "other");
- readAssertEquals(store, "test", 2);
- readAssertEquals(store, "other", 5);
- LOG.info("Updating test");
- try (AtomicOutputStream out = store.updateBlob("test", null)) {
- out.write(3);
+ // Check for Blobstore with authentication
+ public void testWithAuthentication(BlobStore store) throws Exception {
+ //Test for Nimbus Admin
+ Subject admin = getSubject("admin");
+ assertStoreHasExactly(store);
+ SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, admin)) {
+ assertStoreHasExactly(store, "test");
+ out.write(1);
+ }
+ store.deleteBlob("test", admin);
+
+ //Test for Supervisor Admin
+ Subject supervisor = getSubject("supervisor");
+ assertStoreHasExactly(store);
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, supervisor)) {
+ assertStoreHasExactly(store, "test");
+ out.write(1);
+ }
+ store.deleteBlob("test", supervisor);
+
+ //Test for Nimbus itself as a user
+ Subject nimbus = getNimbusSubject();
+ assertStoreHasExactly(store);
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, nimbus)) {
+ assertStoreHasExactly(store, "test");
+ out.write(1);
+ }
+ store.deleteBlob("test", nimbus);
+
+ // Test with a dummy test_subject for cases where subject !=null (security turned on)
+ Subject who = getSubject("test_subject");
+ assertStoreHasExactly(store);
+
+ // Tests for case when subject != null (security turned on) and
+ // acls for the blob are set to WORLD_EVERYTHING
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
+ out.write(1);
+ }
+ assertStoreHasExactly(store, "test");
+ // Testing whether acls are set to WORLD_EVERYTHING
+ assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+ readAssertEqualsWithAuth(store, who, "test", 1);
+
+ LOG.info("Deleting test");
+ store.deleteBlob("test", who);
+ assertStoreHasExactly(store);
+
+ // Tests for case when subject != null (security turned on) and
+ // acls are not set for the blob (DEFAULT)
+ LOG.info("Creating test again");
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test");
+ // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain WORLD_EVERYTHING because
+ // the subject is neither null nor empty. The ACL should however contain USER_EVERYTHING as user needs to have
+ // complete access to the blob
+ assertTrue("ACL does not contain WORLD_EVERYTHING", !metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+ readAssertEqualsWithAuth(store, who, "test", 2);
+
+ LOG.info("Updating test");
+ try (AtomicOutputStream out = store.updateBlob("test", who)) {
+ out.write(3);
+ }
+ assertStoreHasExactly(store, "test");
+ readAssertEqualsWithAuth(store, who, "test", 3);
+
+ LOG.info("Updating test again");
+ try (AtomicOutputStream out = store.updateBlob("test", who)) {
+ out.write(4);
+ out.flush();
+ LOG.info("SLEEPING");
+ Thread.sleep(2);
+ assertStoreHasExactly(store, "test");
+ readAssertEqualsWithAuth(store, who, "test", 3);
+ }
+
+ // Test for subject with no principals and acls set to WORLD_EVERYTHING
+ who = new Subject();
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
+ LOG.info("Creating test");
+ try (AtomicOutputStream out = store.createBlob("test-empty-subject-WE", metadata, who)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test-empty-subject-WE", "test");
+ // Testing whether acls are set to WORLD_EVERYTHING
+ assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+ readAssertEqualsWithAuth(store, who, "test-empty-subject-WE", 2);
+
+ // Test for subject with no principals and acls set to DEFAULT
+ who = new Subject();
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ LOG.info("Creating other");
+
+ try (AtomicOutputStream out = store.createBlob("test-empty-subject-DEF", metadata, who)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test-empty-subject-DEF", "test", "test-empty-subject-WE");
+ // Testing whether acls are set to WORLD_EVERYTHING
+ assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+ readAssertEqualsWithAuth(store, who, "test-empty-subject-DEF", 2);
+
+ if (store instanceof LocalFsBlobStore) {
+ ((LocalFsBlobStore) store).fullCleanup(1);
+ } else {
+ fail("Error the blobstore is of unknowntype");
+ }
}
- assertStoreHasExactly(store, "test", "other");
- readAssertEquals(store, "test", 3);
- readAssertEquals(store, "other", 5);
-
- LOG.info("Deleting other");
- store.deleteBlob("other", null);
- assertStoreHasExactly(store, "test");
- readAssertEquals(store, "test", 3);
-
- LOG.info("Updating test again");
-
- // intended to not guarding with try-with-resource since otherwise test will fail
- AtomicOutputStream out = store.updateBlob("test", null);
- out.write(4);
- out.flush();
- LOG.info("SLEEPING");
- Thread.sleep(2);
-
- if (store instanceof LocalFsBlobStore) {
- ((LocalFsBlobStore) store).fullCleanup(1);
- } else {
- fail("Error the blobstore is of unknowntype");
- } assertStoreHasExactly(store, "test");
- readAssertEquals(store, "test", 3);
- try {
- out.close();
- } catch (IOException e) {
- // This is likely to happen when we try to commit something that
- // was cleaned up. This is expected and acceptable.
+
+ public void testBasic(BlobStore store) throws Exception {
+ assertStoreHasExactly(store);
+ LOG.info("Creating test");
+ // Tests for case when subject == null (security turned off) and
+ // acls for the blob are set to WORLD_EVERYTHING
+ SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler
+ .WORLD_EVERYTHING);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, null)) {
+ out.write(1);
+ }
+ assertStoreHasExactly(store, "test");
+ // Testing whether acls are set to WORLD_EVERYTHING
+ assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+ readAssertEquals(store, "test", 1);
+
+ LOG.info("Deleting test");
+ store.deleteBlob("test", null);
+ assertStoreHasExactly(store);
+
+ // The following tests are run for both hdfs and local store to test the
+ // update blob interface
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
+ LOG.info("Creating test again");
+ try (AtomicOutputStream out = store.createBlob("test", metadata, null)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test");
+ if (store instanceof LocalFsBlobStore) {
+ assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+ }
+ readAssertEquals(store, "test", 2);
+ LOG.info("Updating test");
+ try (AtomicOutputStream out = store.updateBlob("test", null)) {
+ out.write(3);
+ }
+ assertStoreHasExactly(store, "test");
+ readAssertEquals(store, "test", 3);
+
+ LOG.info("Updating test again");
+ try (AtomicOutputStream out = store.updateBlob("test", null)) {
+ out.write(4);
+ out.flush();
+ LOG.info("SLEEPING");
+ Thread.sleep(2);
+ }
+
+ // Tests for case when subject == null (security turned off) and
+ // acls for the blob are set to DEFAULT (Empty ACL List) only for LocalFsBlobstore
+ if (store instanceof LocalFsBlobStore) {
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ LOG.info("Creating test for empty acls when security is off");
+ try (AtomicOutputStream out = store.createBlob("test-empty-acls", metadata, null)) {
+ LOG.info("metadata {}", metadata);
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test-empty-acls", "test");
+ // Testing whether acls are set to WORLD_EVERYTHING, Here we are testing only for LocalFsBlobstore
+ // as the HdfsBlobstore gets the subject information of the local system user and behaves as it is
+ // always authenticated.
+ assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.get_acl().toString().contains("OTHER"));
+
+ LOG.info("Deleting test-empty-acls");
+ store.deleteBlob("test-empty-acls", null);
+ }
+
+ if (store instanceof LocalFsBlobStore) {
+ ((LocalFsBlobStore) store).fullCleanup(1);
+ } else {
+ fail("Error the blobstore is of unknowntype");
+ }
}
- }
-
- @Test
- public void testGetFileLength()
- throws AuthorizationException, KeyNotFoundException, KeyAlreadyExistsException, IOException {
- LocalFsBlobStore store = initLocalFs();
- try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler
- .WORLD_EVERYTHING), null)) {
- out.write(1);
+
+
+ public void testMultiple(BlobStore store) throws Exception {
+
+ assertStoreHasExactly(store);
+ LOG.info("Creating test");
+ try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler
+ .WORLD_EVERYTHING), null)) {
+ out.write(1);
+ }
+ assertStoreHasExactly(store, "test");
+ readAssertEquals(store, "test", 1);
+
+ LOG.info("Creating other");
+ try (AtomicOutputStream out = store.createBlob("other", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
+ null)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test", "other");
+ readAssertEquals(store, "test", 1);
+ readAssertEquals(store, "other", 2);
+
+ LOG.info("Updating other");
+ try (AtomicOutputStream out = store.updateBlob("other", null)) {
+ out.write(5);
+ }
+ assertStoreHasExactly(store, "test", "other");
+ readAssertEquals(store, "test", 1);
+ readAssertEquals(store, "other", 5);
+
+ LOG.info("Deleting test");
+ store.deleteBlob("test", null);
+ assertStoreHasExactly(store, "other");
+ readAssertEquals(store, "other", 5);
+
+ LOG.info("Creating test again");
+ try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
+ null)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test", "other");
+ readAssertEquals(store, "test", 2);
+ readAssertEquals(store, "other", 5);
+
+ LOG.info("Updating test");
+ try (AtomicOutputStream out = store.updateBlob("test", null)) {
+ out.write(3);
+ }
+ assertStoreHasExactly(store, "test", "other");
+ readAssertEquals(store, "test", 3);
+ readAssertEquals(store, "other", 5);
+
+ LOG.info("Deleting other");
+ store.deleteBlob("other", null);
+ assertStoreHasExactly(store, "test");
+ readAssertEquals(store, "test", 3);
+
+ LOG.info("Updating test again");
+
+ // intended to not guarding with try-with-resource since otherwise test will fail
+ AtomicOutputStream out = store.updateBlob("test", null);
+ out.write(4);
+ out.flush();
+ LOG.info("SLEEPING");
+ Thread.sleep(2);
+
+ if (store instanceof LocalFsBlobStore) {
+ ((LocalFsBlobStore) store).fullCleanup(1);
+ } else {
+ fail("Error the blobstore is of unknowntype");
+ }
+ assertStoreHasExactly(store, "test");
+ readAssertEquals(store, "test", 3);
+ try {
+ out.close();
+ } catch (IOException e) {
+ // This is likely to happen when we try to commit something that
+ // was cleaned up. This is expected and acceptable.
+ }
}
- try (InputStreamWithMeta blobInputStream = store.getBlob("test", null)) {
- assertEquals(1, blobInputStream.getFileLength());
+
+ @Test
+ public void testGetFileLength()
+ throws AuthorizationException, KeyNotFoundException, KeyAlreadyExistsException, IOException {
+ LocalFsBlobStore store = initLocalFs();
+ try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler
+ .WORLD_EVERYTHING), null)) {
+ out.write(1);
+ }
+ try (InputStreamWithMeta blobInputStream = store.getBlob("test", null)) {
+ assertEquals(1, blobInputStream.getFileLength());
+ }
}
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreUtilsTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreUtilsTest.java b/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreUtilsTest.java
index e861db8..2c3a49a 100644
--- a/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreUtilsTest.java
+++ b/storm-server/src/test/java/org/apache/storm/blobstore/BlobStoreUtilsTest.java
@@ -1,22 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.blobstore;
+import java.util.List;
+import java.util.Map;
+import org.apache.storm.nimbus.NimbusInfo;
+import org.junit.Test;
+
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.mock;
@@ -24,19 +24,13 @@ import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.storm.nimbus.NimbusInfo;
-import org.junit.Test;
-
public class BlobStoreUtilsTest {
private static final String KEY = "key";
private static final String BLOBSTORE_KEY = "/blobstore/" + KEY;
@SuppressWarnings("unchecked")
- private Map<String, Object> conf = (Map<String, Object>)mock(Map.class);
+ private Map<String, Object> conf = (Map<String, Object>) mock(Map.class);
private MockZookeeperClientBuilder zkClientBuilder = new MockZookeeperClientBuilder();
private BlobStore blobStore = mock(BlobStore.class);
private NimbusInfo nimbusDetails = mock(NimbusInfo.class);
@@ -75,7 +69,7 @@ public class BlobStoreUtilsTest {
@Test
public void testUpdateKeyForBlobStore_nodeWithNullChildren() {
zkClientBuilder.withExists(BLOBSTORE_KEY, true);
- zkClientBuilder.withGetChildren(BLOBSTORE_KEY, (List<String>)null);
+ zkClientBuilder.withGetChildren(BLOBSTORE_KEY, (List<String>) null);
BlobStoreUtils.updateKeyForBlobStore(conf, blobStore, zkClientBuilder.build(), KEY, nimbusDetails);
zkClientBuilder.verifyExists(true);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/blobstore/BlobSynchronizerTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/BlobSynchronizerTest.java b/storm-server/src/test/java/org/apache/storm/blobstore/BlobSynchronizerTest.java
index ea12204..d988b48 100644
--- a/storm-server/src/test/java/org/apache/storm/blobstore/BlobSynchronizerTest.java
+++ b/storm-server/src/test/java/org/apache/storm/blobstore/BlobSynchronizerTest.java
@@ -1,40 +1,40 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.blobstore;
-import org.apache.storm.Config;
-import org.apache.storm.nimbus.NimbusInfo;
-import org.apache.storm.utils.Utils;
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
import org.apache.commons.io.FileUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.curator.test.TestingServer;
+import org.apache.storm.Config;
+import org.apache.storm.nimbus.NimbusInfo;
+import org.apache.storm.utils.Utils;
import org.apache.zookeeper.server.NIOServerCnxnFactory;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.util.*;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -43,93 +43,94 @@ import static org.junit.Assert.assertTrue;
* and BlobSynchronizer class methods
*/
public class BlobSynchronizerTest {
- private URI base;
- private File baseFile;
- private static Map<String, Object> conf = new HashMap();
- private NIOServerCnxnFactory factory;
+ private static Map<String, Object> conf = new HashMap();
+ private URI base;
+ private File baseFile;
+ private NIOServerCnxnFactory factory;
- @Before
- public void init() throws Exception {
- initializeConfigs();
- baseFile = new File("target/blob-store-test-"+ UUID.randomUUID());
- base = baseFile.toURI();
- }
+ // Method which initializes nimbus admin
+ public static void initializeConfigs() {
+ conf.put(Config.NIMBUS_ADMINS, "admin");
+ conf.put(Config.NIMBUS_SUPERVISOR_USERS, "supervisor");
+ }
- @After
- public void cleanUp() throws IOException {
- FileUtils.deleteDirectory(baseFile);
- if (factory != null) {
- factory.shutdown();
+ @Before
+ public void init() throws Exception {
+ initializeConfigs();
+ baseFile = new File("target/blob-store-test-" + UUID.randomUUID());
+ base = baseFile.toURI();
}
- }
- // Method which initializes nimbus admin
- public static void initializeConfigs() {
- conf.put(Config.NIMBUS_ADMINS,"admin");
- conf.put(Config.NIMBUS_SUPERVISOR_USERS,"supervisor");
- }
+ @After
+ public void cleanUp() throws IOException {
+ FileUtils.deleteDirectory(baseFile);
+ if (factory != null) {
+ factory.shutdown();
+ }
+ }
- private LocalFsBlobStore initLocalFs() {
- LocalFsBlobStore store = new LocalFsBlobStore();
- Map<String, Object> conf = Utils.readStormConfig();
- conf.put(Config.STORM_LOCAL_DIR, baseFile.getAbsolutePath());
- conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,"org.apache.storm.security.auth.DefaultPrincipalToLocal");
- this.conf = conf;
- store.prepare(conf, null, null);
- return store;
- }
+ private LocalFsBlobStore initLocalFs() {
+ LocalFsBlobStore store = new LocalFsBlobStore();
+ Map<String, Object> conf = Utils.readStormConfig();
+ conf.put(Config.STORM_LOCAL_DIR, baseFile.getAbsolutePath());
+ conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal");
+ this.conf = conf;
+ store.prepare(conf, null, null);
+ return store;
+ }
- @Test
- public void testBlobSynchronizerForKeysToDownload() {
- BlobStore store = initLocalFs();
- BlobSynchronizer sync = new BlobSynchronizer(store, conf);
- // test for keylist to download
- Set<String> zkSet = new HashSet<String>();
- zkSet.add("key1");
- Set<String> blobStoreSet = new HashSet<String>();
- blobStoreSet.add("key1");
- Set<String> resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet);
- assertTrue("Not Empty", resultSet.isEmpty());
- zkSet.add("key1");
- blobStoreSet.add("key2");
- resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet);
- assertTrue("Not Empty", resultSet.isEmpty());
- blobStoreSet.remove("key1");
- blobStoreSet.remove("key2");
- zkSet.add("key1");
- resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet);
- assertTrue("Unexpected keys to download", (resultSet.size() == 1) && (resultSet.contains("key1")));
- }
+ @Test
+ public void testBlobSynchronizerForKeysToDownload() {
+ BlobStore store = initLocalFs();
+ BlobSynchronizer sync = new BlobSynchronizer(store, conf);
+ // test for keylist to download
+ Set<String> zkSet = new HashSet<String>();
+ zkSet.add("key1");
+ Set<String> blobStoreSet = new HashSet<String>();
+ blobStoreSet.add("key1");
+ Set<String> resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet);
+ assertTrue("Not Empty", resultSet.isEmpty());
+ zkSet.add("key1");
+ blobStoreSet.add("key2");
+ resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet);
+ assertTrue("Not Empty", resultSet.isEmpty());
+ blobStoreSet.remove("key1");
+ blobStoreSet.remove("key2");
+ zkSet.add("key1");
+ resultSet = sync.getKeySetToDownload(blobStoreSet, zkSet);
+ assertTrue("Unexpected keys to download", (resultSet.size() == 1) && (resultSet.contains("key1")));
+ }
- @Test
- public void testGetLatestSequenceNumber() throws Exception {
- List<String> stateInfoList = new ArrayList<String>();
- stateInfoList.add("nimbus1:8000-2");
- stateInfoList.add("nimbus-1:8000-4");
- assertTrue("Failed to get the latest version", BlobStoreUtils.getLatestSequenceNumber(stateInfoList)==4);
- }
+ @Test
+ public void testGetLatestSequenceNumber() throws Exception {
+ List<String> stateInfoList = new ArrayList<String>();
+ stateInfoList.add("nimbus1:8000-2");
+ stateInfoList.add("nimbus-1:8000-4");
+ assertTrue("Failed to get the latest version", BlobStoreUtils.getLatestSequenceNumber(stateInfoList) == 4);
+ }
- @Test
- public void testNimbodesWithLatestVersionOfBlob() throws Exception {
- try (TestingServer server = new TestingServer(); CuratorFramework zkClient = CuratorFrameworkFactory.newClient(server.getConnectString(), new ExponentialBackoffRetry(1000, 3))) {
- zkClient.start();
- // Creating nimbus hosts containing latest version of blob
- zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1");
- zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2");
- Set<NimbusInfo> set = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, "key1");
- assertEquals("Failed to get the correct nimbus hosts with latest blob version", (set.iterator().next()).getHost(),"nimbus2");
- zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1");
- zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2");
- }
- }
+ @Test
+ public void testNimbodesWithLatestVersionOfBlob() throws Exception {
+ try (TestingServer server = new TestingServer(); CuratorFramework zkClient = CuratorFrameworkFactory
+ .newClient(server.getConnectString(), new ExponentialBackoffRetry(1000, 3))) {
+ zkClient.start();
+ // Creating nimbus hosts containing latest version of blob
+ zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1");
+ zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2");
+ Set<NimbusInfo> set = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, "key1");
+ assertEquals("Failed to get the correct nimbus hosts with latest blob version", (set.iterator().next()).getHost(), "nimbus2");
+ zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1");
+ zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2");
+ }
+ }
- @Test
- public void testNormalizeVersionInfo () throws Exception {
- BlobKeySequenceInfo info1 = BlobStoreUtils.normalizeNimbusHostPortSequenceNumberInfo("nimbus1:7800-1");
- assertTrue(info1.getNimbusHostPort().equals("nimbus1:7800"));
- assertTrue(info1.getSequenceNumber().equals("1"));
- BlobKeySequenceInfo info2 = BlobStoreUtils.normalizeNimbusHostPortSequenceNumberInfo("nimbus-1:7800-1");
- assertTrue(info2.getNimbusHostPort().equals("nimbus-1:7800"));
- assertTrue(info2.getSequenceNumber().equals("1"));
- }
+ @Test
+ public void testNormalizeVersionInfo() throws Exception {
+ BlobKeySequenceInfo info1 = BlobStoreUtils.normalizeNimbusHostPortSequenceNumberInfo("nimbus1:7800-1");
+ assertTrue(info1.getNimbusHostPort().equals("nimbus1:7800"));
+ assertTrue(info1.getSequenceNumber().equals("1"));
+ BlobKeySequenceInfo info2 = BlobStoreUtils.normalizeNimbusHostPortSequenceNumberInfo("nimbus-1:7800-1");
+ assertTrue(info2.getNimbusHostPort().equals("nimbus-1:7800"));
+ assertTrue(info2.getSequenceNumber().equals("1"));
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java b/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java
index c3e2b84..2da3b2d 100644
--- a/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java
+++ b/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java
@@ -1,30 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.blobstore;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+package org.apache.storm.blobstore;
import java.util.Arrays;
import java.util.List;
-
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.ExistsBuilder;
import org.apache.curator.framework.api.GetChildrenBuilder;
@@ -32,6 +21,11 @@ import org.apache.curator.framework.api.Pathable;
import org.apache.log4j.Logger;
import org.apache.zookeeper.data.Stat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
public class MockZookeeperClientBuilder {
private static final Logger LOG = Logger.getLogger(MockZookeeperClientBuilder.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/daemon/drpc/DRPCTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/daemon/drpc/DRPCTest.java b/storm-server/src/test/java/org/apache/storm/daemon/drpc/DRPCTest.java
index 73f5ff4..dee13dc 100644
--- a/storm-server/src/test/java/org/apache/storm/daemon/drpc/DRPCTest.java
+++ b/storm-server/src/test/java/org/apache/storm/daemon/drpc/DRPCTest.java
@@ -15,9 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.daemon.drpc;
-import static org.junit.Assert.*;
+package org.apache.storm.daemon.drpc;
import java.util.Arrays;
import java.util.HashMap;
@@ -27,9 +26,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-
import javax.security.auth.Subject;
-
import org.apache.storm.Config;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.DRPCExceptionType;
@@ -39,19 +36,20 @@ import org.apache.storm.security.auth.DefaultPrincipalToLocal;
import org.apache.storm.security.auth.ReqContext;
import org.apache.storm.security.auth.SingleUserPrincipal;
import org.apache.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer;
-import org.apache.storm.security.auth.authorizer.DenyAuthorizer;
import org.apache.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer.AclFunctionEntry;
+import org.apache.storm.security.auth.authorizer.DenyAuthorizer;
import org.apache.storm.utils.Time;
import org.junit.AfterClass;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
public class DRPCTest {
private static final ExecutorService exec = Executors.newCachedThreadPool();
-
- public static interface ThrowStuff {
- public void run() throws Exception;
- }
-
+
private static void assertThrows(ThrowStuff t, Class<? extends Exception> expected) {
try {
t.run();
@@ -60,12 +58,12 @@ public class DRPCTest {
assertTrue("Expected " + t + " to throw " + expected + " but threw " + e, expected.isInstance(e));
}
}
-
+
@AfterClass
public static void close() {
exec.shutdownNow();
}
-
+
public static DRPCRequest getNextAvailableRequest(DRPC server, String func) throws Exception {
DRPCRequest request = null;
long timedout = System.currentTimeMillis() + 5_000;
@@ -79,7 +77,7 @@ public class DRPCTest {
fail("Test timed out waiting for a request on " + func);
return request;
}
-
+
@Test
public void testGoodBlocking() throws Exception {
try (DRPC server = new DRPC(null, 100)) {
@@ -93,7 +91,7 @@ public class DRPCTest {
assertEquals("tested", result);
}
}
-
+
@Test
public void testFailedBlocking() throws Exception {
try (DRPC server = new DRPC(null, 100)) {
@@ -110,11 +108,11 @@ public class DRPCTest {
Throwable t = e.getCause();
assertEquals(t.getClass(), DRPCExecutionException.class);
//Don't know a better way to validate that it failed.
- assertEquals(DRPCExceptionType.FAILED_REQUEST, ((DRPCExecutionException)t).get_type());
+ assertEquals(DRPCExceptionType.FAILED_REQUEST, ((DRPCExecutionException) t).get_type());
}
}
}
-
+
@Test
public void testDequeueAfterTimeout() throws Exception {
long timeout = 1000;
@@ -135,7 +133,7 @@ public class DRPCTest {
assertEquals("", request.get_func_args());
}
}
-
+
@Test
public void testDeny() throws Exception {
try (DRPC server = new DRPC(new DenyAuthorizer(), 100)) {
@@ -143,7 +141,7 @@ public class DRPCTest {
assertThrows(() -> server.fetchRequest("testing"), AuthorizationException.class);
}
}
-
+
@Test
public void testStrict() throws Exception {
ReqContext jt = new ReqContext(new Subject());
@@ -174,29 +172,29 @@ public class DRPCTest {
DRPC.checkAuthorization(jt, auth, "fetchRequest", "jump");
assertThrows(() -> DRPC.checkAuthorization(jc, auth, "fetchRequest", "jump"), AuthorizationException.class);
assertThrows(() -> DRPC.checkAuthorization(other, auth, "fetchRequest", "jump"), AuthorizationException.class);
-
+
DRPC.checkAuthorization(jt, auth, "result", "jump");
assertThrows(() -> DRPC.checkAuthorization(jc, auth, "result", "jump"), AuthorizationException.class);
assertThrows(() -> DRPC.checkAuthorization(other, auth, "result", "jump"), AuthorizationException.class);
-
+
assertThrows(() -> DRPC.checkAuthorization(jt, auth, "execute", "jump"), AuthorizationException.class);
DRPC.checkAuthorization(jc, auth, "execute", "jump");
assertThrows(() -> DRPC.checkAuthorization(other, auth, "execute", "jump"), AuthorizationException.class);
-
+
//not_jump (closed in strict mode)
assertThrows(() -> DRPC.checkAuthorization(jt, auth, "fetchRequest", "not_jump"), AuthorizationException.class);
assertThrows(() -> DRPC.checkAuthorization(jc, auth, "fetchRequest", "not_jump"), AuthorizationException.class);
assertThrows(() -> DRPC.checkAuthorization(other, auth, "fetchRequest", "not_jump"), AuthorizationException.class);
-
+
assertThrows(() -> DRPC.checkAuthorization(jt, auth, "result", "not_jump"), AuthorizationException.class);
assertThrows(() -> DRPC.checkAuthorization(jc, auth, "result", "not_jump"), AuthorizationException.class);
assertThrows(() -> DRPC.checkAuthorization(other, auth, "result", "not_jump"), AuthorizationException.class);
-
+
assertThrows(() -> DRPC.checkAuthorization(jt, auth, "execute", "not_jump"), AuthorizationException.class);
assertThrows(() -> DRPC.checkAuthorization(jc, auth, "execute", "not_jump"), AuthorizationException.class);
assertThrows(() -> DRPC.checkAuthorization(other, auth, "execute", "not_jump"), AuthorizationException.class);
}
-
+
@Test
public void testNotStrict() throws Exception {
ReqContext jt = new ReqContext(new Subject());
@@ -227,26 +225,30 @@ public class DRPCTest {
DRPC.checkAuthorization(jt, auth, "fetchRequest", "jump");
assertThrows(() -> DRPC.checkAuthorization(jc, auth, "fetchRequest", "jump"), AuthorizationException.class);
assertThrows(() -> DRPC.checkAuthorization(other, auth, "fetchRequest", "jump"), AuthorizationException.class);
-
+
DRPC.checkAuthorization(jt, auth, "result", "jump");
assertThrows(() -> DRPC.checkAuthorization(jc, auth, "result", "jump"), AuthorizationException.class);
assertThrows(() -> DRPC.checkAuthorization(other, auth, "result", "jump"), AuthorizationException.class);
-
+
assertThrows(() -> DRPC.checkAuthorization(jt, auth, "execute", "jump"), AuthorizationException.class);
DRPC.checkAuthorization(jc, auth, "execute", "jump");
assertThrows(() -> DRPC.checkAuthorization(other, auth, "execute", "jump"), AuthorizationException.class);
-
+
//not_jump (open in not strict mode)
DRPC.checkAuthorization(jt, auth, "fetchRequest", "not_jump");
DRPC.checkAuthorization(jc, auth, "fetchRequest", "not_jump");
DRPC.checkAuthorization(other, auth, "fetchRequest", "not_jump");
-
+
DRPC.checkAuthorization(jt, auth, "result", "not_jump");
DRPC.checkAuthorization(jc, auth, "result", "not_jump");
DRPC.checkAuthorization(other, auth, "result", "not_jump");
-
+
DRPC.checkAuthorization(jt, auth, "execute", "not_jump");
DRPC.checkAuthorization(jc, auth, "execute", "not_jump");
DRPC.checkAuthorization(other, auth, "execute", "not_jump");
}
+
+ public static interface ThrowStuff {
+ public void run() throws Exception;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusTest.java b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusTest.java
index 60ff2d7..47484ac 100644
--- a/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusTest.java
+++ b/storm-server/src/test/java/org/apache/storm/daemon/nimbus/NimbusTest.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.daemon.nimbus;
import org.apache.storm.Config;
@@ -26,7 +27,7 @@ import org.apache.storm.testing.TestWordSpout;
import org.apache.storm.topology.TopologyBuilder;
import org.junit.Test;
-import static org.junit.Assert.*;
+import static org.junit.Assert.fail;
public class NimbusTest {
@Test
[39/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-starter
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java
index faa4e32..d7ab065 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java
@@ -1,24 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.tools;
-import org.apache.storm.utils.Time;
import org.apache.commons.collections.buffer.CircularFifoBuffer;
+import org.apache.storm.utils.Time;
/**
* This class tracks the time-since-last-modify of a "thing" in a rolling fashion.
@@ -30,41 +25,41 @@ import org.apache.commons.collections.buffer.CircularFifoBuffer;
*/
public class NthLastModifiedTimeTracker {
- private static final int MILLIS_IN_SEC = 1000;
+ private static final int MILLIS_IN_SEC = 1000;
- private final CircularFifoBuffer lastModifiedTimesMillis;
+ private final CircularFifoBuffer lastModifiedTimesMillis;
- public NthLastModifiedTimeTracker(int numTimesToTrack) {
- if (numTimesToTrack < 1) {
- throw new IllegalArgumentException(
- "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack + ")");
+ public NthLastModifiedTimeTracker(int numTimesToTrack) {
+ if (numTimesToTrack < 1) {
+ throw new IllegalArgumentException(
+ "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack + ")");
+ }
+ lastModifiedTimesMillis = new CircularFifoBuffer(numTimesToTrack);
+ initLastModifiedTimesMillis();
}
- lastModifiedTimesMillis = new CircularFifoBuffer(numTimesToTrack);
- initLastModifiedTimesMillis();
- }
- private void initLastModifiedTimesMillis() {
- long nowCached = now();
- for (int i = 0; i < lastModifiedTimesMillis.maxSize(); i++) {
- lastModifiedTimesMillis.add(Long.valueOf(nowCached));
+ private void initLastModifiedTimesMillis() {
+ long nowCached = now();
+ for (int i = 0; i < lastModifiedTimesMillis.maxSize(); i++) {
+ lastModifiedTimesMillis.add(Long.valueOf(nowCached));
+ }
}
- }
- private long now() {
- return Time.currentTimeMillis();
- }
+ private long now() {
+ return Time.currentTimeMillis();
+ }
- public int secondsSinceOldestModification() {
- long modifiedTimeMillis = ((Long) lastModifiedTimesMillis.get()).longValue();
- return (int) ((now() - modifiedTimeMillis) / MILLIS_IN_SEC);
- }
+ public int secondsSinceOldestModification() {
+ long modifiedTimeMillis = ((Long) lastModifiedTimesMillis.get()).longValue();
+ return (int) ((now() - modifiedTimeMillis) / MILLIS_IN_SEC);
+ }
- public void markAsModified() {
- updateLastModifiedTime();
- }
+ public void markAsModified() {
+ updateLastModifiedTime();
+ }
- private void updateLastModifiedTime() {
- lastModifiedTimesMillis.add(now());
- }
+ private void updateLastModifiedTime() {
+ lastModifiedTimesMillis.add(now());
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java
index 85f2b62..ea9e6d6 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankable.java
@@ -1,32 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.tools;
public interface Rankable extends Comparable<Rankable> {
- Object getObject();
+ Object getObject();
- long getCount();
+ long getCount();
- /**
- * Note: We do not defensively copy the object wrapped by the Rankable. It is passed as is.
- *
- * @return a defensive copy
- */
- Rankable copy();
+ /**
+ * Note: We do not defensively copy the object wrapped by the Rankable. It is passed as is.
+ *
+ * @return a defensive copy
+ */
+ Rankable copy();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java
index b1a9dca..0b3808e 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/RankableObjectWithFields.java
@@ -1,28 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.tools;
-import org.apache.storm.tuple.Tuple;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-
import java.io.Serializable;
import java.util.List;
+import org.apache.storm.tuple.Tuple;
/**
* This class wraps an objects and its associated count, including any additional data fields.
@@ -31,118 +25,116 @@ import java.util.List;
*/
public class RankableObjectWithFields implements Rankable, Serializable {
- private static final long serialVersionUID = -9102878650001058090L;
- private static final String toStringSeparator = "|";
+ private static final long serialVersionUID = -9102878650001058090L;
+ private static final String toStringSeparator = "|";
- private final Object obj;
- private final long count;
- private final ImmutableList<Object> fields;
+ private final Object obj;
+ private final long count;
+ private final ImmutableList<Object> fields;
+
+ public RankableObjectWithFields(Object obj, long count, Object... otherFields) {
+ if (obj == null) {
+ throw new IllegalArgumentException("The object must not be null");
+ }
+ if (count < 0) {
+ throw new IllegalArgumentException("The count must be >= 0");
+ }
+ this.obj = obj;
+ this.count = count;
+ fields = ImmutableList.copyOf(otherFields);
- public RankableObjectWithFields(Object obj, long count, Object... otherFields) {
- if (obj == null) {
- throw new IllegalArgumentException("The object must not be null");
}
- if (count < 0) {
- throw new IllegalArgumentException("The count must be >= 0");
+
+ /**
+ * Construct a new instance based on the provided {@link Tuple}.
+ * <p/>
+ * This method expects the object to be ranked in the first field (index 0) of the provided tuple, and the number of
+ * occurrences of the object (its count) in the second field (index 1). Any further fields in the tuple will be
+ * extracted and tracked, too. These fields can be accessed via {@link RankableObjectWithFields#getFields()}.
+ *
+ * @param tuple
+ *
+ * @return new instance based on the provided tuple
+ */
+ public static RankableObjectWithFields from(Tuple tuple) {
+ List<Object> otherFields = Lists.newArrayList(tuple.getValues());
+ Object obj = otherFields.remove(0);
+ Long count = (Long) otherFields.remove(0);
+ return new RankableObjectWithFields(obj, count, otherFields.toArray());
}
- this.obj = obj;
- this.count = count;
- fields = ImmutableList.copyOf(otherFields);
-
- }
-
- /**
- * Construct a new instance based on the provided {@link Tuple}.
- * <p/>
- * This method expects the object to be ranked in the first field (index 0) of the provided tuple, and the number of
- * occurrences of the object (its count) in the second field (index 1). Any further fields in the tuple will be
- * extracted and tracked, too. These fields can be accessed via {@link RankableObjectWithFields#getFields()}.
- *
- * @param tuple
- *
- * @return new instance based on the provided tuple
- */
- public static RankableObjectWithFields from(Tuple tuple) {
- List<Object> otherFields = Lists.newArrayList(tuple.getValues());
- Object obj = otherFields.remove(0);
- Long count = (Long) otherFields.remove(0);
- return new RankableObjectWithFields(obj, count, otherFields.toArray());
- }
-
- public Object getObject() {
- return obj;
- }
-
- public long getCount() {
- return count;
- }
-
- /**
- * @return an immutable list of any additional data fields of the object (may be empty but will never be null)
- */
- public List<Object> getFields() {
- return fields;
- }
-
- @Override
- public int compareTo(Rankable other) {
- long delta = this.getCount() - other.getCount();
- if (delta > 0) {
- return 1;
+
+ public Object getObject() {
+ return obj;
}
- else if (delta < 0) {
- return -1;
+
+ public long getCount() {
+ return count;
}
- else {
- return 0;
+
+ /**
+ * @return an immutable list of any additional data fields of the object (may be empty but will never be null)
+ */
+ public List<Object> getFields() {
+ return fields;
+ }
+
+ @Override
+ public int compareTo(Rankable other) {
+ long delta = this.getCount() - other.getCount();
+ if (delta > 0) {
+ return 1;
+ } else if (delta < 0) {
+ return -1;
+ } else {
+ return 0;
+ }
}
- }
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof RankableObjectWithFields)) {
+ return false;
+ }
+ RankableObjectWithFields other = (RankableObjectWithFields) o;
+ return obj.equals(other.obj) && count == other.count;
}
- if (!(o instanceof RankableObjectWithFields)) {
- return false;
+
+ @Override
+ public int hashCode() {
+ int result = 17;
+ int countHash = (int) (count ^ (count >>> 32));
+ result = 31 * result + countHash;
+ result = 31 * result + obj.hashCode();
+ return result;
}
- RankableObjectWithFields other = (RankableObjectWithFields) o;
- return obj.equals(other.obj) && count == other.count;
- }
-
- @Override
- public int hashCode() {
- int result = 17;
- int countHash = (int) (count ^ (count >>> 32));
- result = 31 * result + countHash;
- result = 31 * result + obj.hashCode();
- return result;
- }
-
- public String toString() {
- StringBuffer buf = new StringBuffer();
- buf.append("[");
- buf.append(obj);
- buf.append(toStringSeparator);
- buf.append(count);
- for (Object field : fields) {
- buf.append(toStringSeparator);
- buf.append(field);
+
+ public String toString() {
+ StringBuffer buf = new StringBuffer();
+ buf.append("[");
+ buf.append(obj);
+ buf.append(toStringSeparator);
+ buf.append(count);
+ for (Object field : fields) {
+ buf.append(toStringSeparator);
+ buf.append(field);
+ }
+ buf.append("]");
+ return buf.toString();
+ }
+
+ /**
+ * Note: We do not defensively copy the wrapped object and any accompanying fields. We do guarantee, however,
+ * do return a defensive (shallow) copy of the List object that is wrapping any accompanying fields.
+ *
+ * @return
+ */
+ @Override
+ public Rankable copy() {
+ List<Object> shallowCopyOfFields = ImmutableList.copyOf(getFields());
+ return new RankableObjectWithFields(getObject(), getCount(), shallowCopyOfFields);
}
- buf.append("]");
- return buf.toString();
- }
-
- /**
- * Note: We do not defensively copy the wrapped object and any accompanying fields. We do guarantee, however,
- * do return a defensive (shallow) copy of the List object that is wrapping any accompanying fields.
- *
- * @return
- */
- @Override
- public Rankable copy() {
- List<Object> shallowCopyOfFields = ImmutableList.copyOf(getFields());
- return new RankableObjectWithFields(getObject(), getCount(), shallowCopyOfFields);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java
index 17174b3..eae6ccd 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/Rankings.java
@@ -1,156 +1,148 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.tools;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-
import java.io.Serializable;
import java.util.Collections;
import java.util.List;
public class Rankings implements Serializable {
- private static final long serialVersionUID = -1549827195410578903L;
- private static final int DEFAULT_COUNT = 10;
+ private static final long serialVersionUID = -1549827195410578903L;
+ private static final int DEFAULT_COUNT = 10;
+
+ private final int maxSize;
+ private final List<Rankable> rankedItems = Lists.newArrayList();
+
+ public Rankings() {
+ this(DEFAULT_COUNT);
+ }
+
+ public Rankings(int topN) {
+ if (topN < 1) {
+ throw new IllegalArgumentException("topN must be >= 1");
+ }
+ maxSize = topN;
+ }
+
+ /**
+ * Copy constructor.
+ * @param other
+ */
+ public Rankings(Rankings other) {
+ this(other.maxSize());
+ updateWith(other);
+ }
- private final int maxSize;
- private final List<Rankable> rankedItems = Lists.newArrayList();
+ /**
+ * @return the maximum possible number (size) of ranked objects this instance can hold
+ */
+ public int maxSize() {
+ return maxSize;
+ }
- public Rankings() {
- this(DEFAULT_COUNT);
- }
+ /**
+ * @return the number (size) of ranked objects this instance is currently holding
+ */
+ public int size() {
+ return rankedItems.size();
+ }
- public Rankings(int topN) {
- if (topN < 1) {
- throw new IllegalArgumentException("topN must be >= 1");
+ /**
+ * The returned defensive copy is only "somewhat" defensive. We do, for instance, return a defensive copy of the
+ * enclosing List instance, and we do try to defensively copy any contained Rankable objects, too. However, the
+ * contract of {@link org.apache.storm.starter.tools.Rankable#copy()} does not guarantee that any Object's embedded within
+ * a Rankable will be defensively copied, too.
+ *
+ * @return a somewhat defensive copy of ranked items
+ */
+ public List<Rankable> getRankings() {
+ List<Rankable> copy = Lists.newLinkedList();
+ for (Rankable r : rankedItems) {
+ copy.add(r.copy());
+ }
+ return ImmutableList.copyOf(copy);
}
- maxSize = topN;
- }
-
- /**
- * Copy constructor.
- * @param other
- */
- public Rankings(Rankings other) {
- this(other.maxSize());
- updateWith(other);
- }
-
- /**
- * @return the maximum possible number (size) of ranked objects this instance can hold
- */
- public int maxSize() {
- return maxSize;
- }
-
- /**
- * @return the number (size) of ranked objects this instance is currently holding
- */
- public int size() {
- return rankedItems.size();
- }
-
- /**
- * The returned defensive copy is only "somewhat" defensive. We do, for instance, return a defensive copy of the
- * enclosing List instance, and we do try to defensively copy any contained Rankable objects, too. However, the
- * contract of {@link org.apache.storm.starter.tools.Rankable#copy()} does not guarantee that any Object's embedded within
- * a Rankable will be defensively copied, too.
- *
- * @return a somewhat defensive copy of ranked items
- */
- public List<Rankable> getRankings() {
- List<Rankable> copy = Lists.newLinkedList();
- for (Rankable r: rankedItems) {
- copy.add(r.copy());
+
+ public void updateWith(Rankings other) {
+ for (Rankable r : other.getRankings()) {
+ updateWith(r);
+ }
}
- return ImmutableList.copyOf(copy);
- }
- public void updateWith(Rankings other) {
- for (Rankable r : other.getRankings()) {
- updateWith(r);
+ public void updateWith(Rankable r) {
+ synchronized (rankedItems) {
+ addOrReplace(r);
+ rerank();
+ shrinkRankingsIfNeeded();
+ }
}
- }
- public void updateWith(Rankable r) {
- synchronized(rankedItems) {
- addOrReplace(r);
- rerank();
- shrinkRankingsIfNeeded();
+ private void addOrReplace(Rankable r) {
+ Integer rank = findRankOf(r);
+ if (rank != null) {
+ rankedItems.set(rank, r);
+ } else {
+ rankedItems.add(r);
+ }
}
- }
- private void addOrReplace(Rankable r) {
- Integer rank = findRankOf(r);
- if (rank != null) {
- rankedItems.set(rank, r);
+ private Integer findRankOf(Rankable r) {
+ Object tag = r.getObject();
+ for (int rank = 0; rank < rankedItems.size(); rank++) {
+ Object cur = rankedItems.get(rank).getObject();
+ if (cur.equals(tag)) {
+ return rank;
+ }
+ }
+ return null;
}
- else {
- rankedItems.add(r);
+
+ private void rerank() {
+ Collections.sort(rankedItems);
+ Collections.reverse(rankedItems);
}
- }
-
- private Integer findRankOf(Rankable r) {
- Object tag = r.getObject();
- for (int rank = 0; rank < rankedItems.size(); rank++) {
- Object cur = rankedItems.get(rank).getObject();
- if (cur.equals(tag)) {
- return rank;
- }
+
+ private void shrinkRankingsIfNeeded() {
+ if (rankedItems.size() > maxSize) {
+ rankedItems.remove(maxSize);
+ }
}
- return null;
- }
- private void rerank() {
- Collections.sort(rankedItems);
- Collections.reverse(rankedItems);
- }
+ /**
+ * Removes ranking entries that have a count of zero.
+ */
+ public void pruneZeroCounts() {
+ int i = 0;
+ while (i < rankedItems.size()) {
+ if (rankedItems.get(i).getCount() == 0) {
+ rankedItems.remove(i);
+ } else {
+ i++;
+ }
+ }
+ }
- private void shrinkRankingsIfNeeded() {
- if (rankedItems.size() > maxSize) {
- rankedItems.remove(maxSize);
+ public String toString() {
+ return rankedItems.toString();
}
- }
-
- /**
- * Removes ranking entries that have a count of zero.
- */
- public void pruneZeroCounts() {
- int i = 0;
- while (i < rankedItems.size()) {
- if (rankedItems.get(i).getCount() == 0) {
- rankedItems.remove(i);
- }
- else {
- i++;
- }
+
+ /**
+ * Creates a (defensive) copy of itself.
+ */
+ public Rankings copy() {
+ return new Rankings(this);
}
- }
-
- public String toString() {
- return rankedItems.toString();
- }
-
- /**
- * Creates a (defensive) copy of itself.
- */
- public Rankings copy() {
- return new Rankings(this);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java
index b95a6a9..6aedfc8 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlidingWindowCounter.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.tools;
import java.io.Serializable;
@@ -67,53 +62,53 @@ import java.util.Map;
*/
public final class SlidingWindowCounter<T> implements Serializable {
- private static final long serialVersionUID = -2645063988768785810L;
+ private static final long serialVersionUID = -2645063988768785810L;
- private SlotBasedCounter<T> objCounter;
- private int headSlot;
- private int tailSlot;
- private int windowLengthInSlots;
+ private SlotBasedCounter<T> objCounter;
+ private int headSlot;
+ private int tailSlot;
+ private int windowLengthInSlots;
- public SlidingWindowCounter(int windowLengthInSlots) {
- if (windowLengthInSlots < 2) {
- throw new IllegalArgumentException(
- "Window length in slots must be at least two (you requested " + windowLengthInSlots + ")");
- }
- this.windowLengthInSlots = windowLengthInSlots;
- this.objCounter = new SlotBasedCounter<T>(this.windowLengthInSlots);
+ public SlidingWindowCounter(int windowLengthInSlots) {
+ if (windowLengthInSlots < 2) {
+ throw new IllegalArgumentException(
+ "Window length in slots must be at least two (you requested " + windowLengthInSlots + ")");
+ }
+ this.windowLengthInSlots = windowLengthInSlots;
+ this.objCounter = new SlotBasedCounter<T>(this.windowLengthInSlots);
- this.headSlot = 0;
- this.tailSlot = slotAfter(headSlot);
- }
+ this.headSlot = 0;
+ this.tailSlot = slotAfter(headSlot);
+ }
- public void incrementCount(T obj) {
- objCounter.incrementCount(obj, headSlot);
- }
+ public void incrementCount(T obj) {
+ objCounter.incrementCount(obj, headSlot);
+ }
- /**
- * Return the current (total) counts of all tracked objects, then advance the window.
- * <p/>
- * Whenever this method is called, we consider the counts of the current sliding window to be available to and
- * successfully processed "upstream" (i.e. by the caller). Knowing this we will start counting any subsequent
- * objects within the next "chunk" of the sliding window.
- *
- * @return The current (total) counts of all tracked objects.
- */
- public Map<T, Long> getCountsThenAdvanceWindow() {
- Map<T, Long> counts = objCounter.getCounts();
- objCounter.wipeZeros();
- objCounter.wipeSlot(tailSlot);
- advanceHead();
- return counts;
- }
+ /**
+ * Return the current (total) counts of all tracked objects, then advance the window.
+ * <p/>
+ * Whenever this method is called, we consider the counts of the current sliding window to be available to and
+ * successfully processed "upstream" (i.e. by the caller). Knowing this we will start counting any subsequent
+ * objects within the next "chunk" of the sliding window.
+ *
+ * @return The current (total) counts of all tracked objects.
+ */
+ public Map<T, Long> getCountsThenAdvanceWindow() {
+ Map<T, Long> counts = objCounter.getCounts();
+ objCounter.wipeZeros();
+ objCounter.wipeSlot(tailSlot);
+ advanceHead();
+ return counts;
+ }
- private void advanceHead() {
- headSlot = tailSlot;
- tailSlot = slotAfter(tailSlot);
- }
+ private void advanceHead() {
+ headSlot = tailSlot;
+ tailSlot = slotAfter(tailSlot);
+ }
- private int slotAfter(int slot) {
- return (slot + 1) % windowLengthInSlots;
- }
+ private int slotAfter(int slot) {
+ return (slot + 1) % windowLengthInSlots;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java
index 5bf66a5..51157b0 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/SlotBasedCounter.java
@@ -1,28 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.tools;
import java.io.Serializable;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
-import java.util.Set;
/**
* This class provides per-slot counts of the occurrences of objects.
@@ -33,84 +26,83 @@ import java.util.Set;
*/
public final class SlotBasedCounter<T> implements Serializable {
- private static final long serialVersionUID = 4858185737378394432L;
+ private static final long serialVersionUID = 4858185737378394432L;
- private final Map<T, long[]> objToCounts = new HashMap<T, long[]>();
- private final int numSlots;
+ private final Map<T, long[]> objToCounts = new HashMap<T, long[]>();
+ private final int numSlots;
- public SlotBasedCounter(int numSlots) {
- if (numSlots <= 0) {
- throw new IllegalArgumentException("Number of slots must be greater than zero (you requested " + numSlots + ")");
+ public SlotBasedCounter(int numSlots) {
+ if (numSlots <= 0) {
+ throw new IllegalArgumentException("Number of slots must be greater than zero (you requested " + numSlots + ")");
+ }
+ this.numSlots = numSlots;
}
- this.numSlots = numSlots;
- }
- public void incrementCount(T obj, int slot) {
- long[] counts = objToCounts.get(obj);
- if (counts == null) {
- counts = new long[this.numSlots];
- objToCounts.put(obj, counts);
+ public void incrementCount(T obj, int slot) {
+ long[] counts = objToCounts.get(obj);
+ if (counts == null) {
+ counts = new long[this.numSlots];
+ objToCounts.put(obj, counts);
+ }
+ counts[slot]++;
}
- counts[slot]++;
- }
- public long getCount(T obj, int slot) {
- long[] counts = objToCounts.get(obj);
- if (counts == null) {
- return 0;
- }
- else {
- return counts[slot];
+ public long getCount(T obj, int slot) {
+ long[] counts = objToCounts.get(obj);
+ if (counts == null) {
+ return 0;
+ } else {
+ return counts[slot];
+ }
}
- }
- public Map<T, Long> getCounts() {
- Map<T, Long> result = new HashMap<T, Long>();
- for (T obj : objToCounts.keySet()) {
- result.put(obj, computeTotalCount(obj));
+ public Map<T, Long> getCounts() {
+ Map<T, Long> result = new HashMap<T, Long>();
+ for (T obj : objToCounts.keySet()) {
+ result.put(obj, computeTotalCount(obj));
+ }
+ return result;
}
- return result;
- }
- private long computeTotalCount(T obj) {
- long[] curr = objToCounts.get(obj);
- long total = 0;
- for (long l : curr) {
- total += l;
+ private long computeTotalCount(T obj) {
+ long[] curr = objToCounts.get(obj);
+ long total = 0;
+ for (long l : curr) {
+ total += l;
+ }
+ return total;
}
- return total;
- }
- /**
- * Reset the slot count of any tracked objects to zero for the given slot.
- *
- * @param slot
- */
- public void wipeSlot(int slot) {
- for (T obj : objToCounts.keySet()) {
- resetSlotCountToZero(obj, slot);
+ /**
+ * Reset the slot count of any tracked objects to zero for the given slot.
+ *
+ * @param slot
+ */
+ public void wipeSlot(int slot) {
+ for (T obj : objToCounts.keySet()) {
+ resetSlotCountToZero(obj, slot);
+ }
}
- }
- private void resetSlotCountToZero(T obj, int slot) {
- long[] counts = objToCounts.get(obj);
- counts[slot] = 0;
- }
+ private void resetSlotCountToZero(T obj, int slot) {
+ long[] counts = objToCounts.get(obj);
+ counts[slot] = 0;
+ }
- private boolean shouldBeRemovedFromCounter(T obj) {
- return computeTotalCount(obj) == 0;
- }
+ private boolean shouldBeRemovedFromCounter(T obj) {
+ return computeTotalCount(obj) == 0;
+ }
- /**
- * Remove any object from the counter whose total count is zero (to free up memory).
- */
- public void wipeZeros() {
- for(Iterator<Map.Entry<T, long[]>> it = objToCounts.entrySet().iterator(); it.hasNext(); ) {
- Map.Entry<T, long[]> entry = it.next();
- if (shouldBeRemovedFromCounter(entry.getKey())) {
- it.remove();
- }
+ /**
+ * Remove any object from the counter whose total count is zero (to free up memory).
+ */
+ public void wipeZeros() {
+ for (Iterator<Map.Entry<T, long[]>> it = objToCounts.entrySet().iterator(); it.hasNext(); ) {
+ Map.Entry<T, long[]> entry = it.next();
+ if (shouldBeRemovedFromCounter(entry.getKey())) {
+ it.remove();
+ }
+ }
}
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java
index 5bb23f7..4849f72 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/DebugMemoryMapState.java
@@ -18,6 +18,9 @@
package org.apache.storm.starter.trident;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
import org.apache.storm.task.IMetricsContext;
import org.apache.storm.topology.FailedException;
import org.apache.storm.trident.state.CombinerValueUpdater;
@@ -28,10 +31,6 @@ import org.apache.storm.trident.testing.MemoryMapState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
public class DebugMemoryMapState<T> extends MemoryMapState<T> {
private static final Logger LOG = LoggerFactory.getLogger(DebugMemoryMapState.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentHBaseWindowingStoreTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentHBaseWindowingStoreTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentHBaseWindowingStoreTopology.java
index fa9274d..6938b8a 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentHBaseWindowingStoreTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentHBaseWindowingStoreTopology.java
@@ -1,25 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.trident;
import java.util.HashMap;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -47,21 +40,23 @@ public class TridentHBaseWindowingStoreTopology {
public static StormTopology buildTopology(WindowsStoreFactory windowsStore) throws Exception {
FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"),
- new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"),
- new Values("how many apples can you eat"), new Values("to be or not to be the person"));
+ new Values("the man went to the store and bought some candy"),
+ new Values("four score and seven years ago"),
+ new Values("how many apples can you eat"), new Values("to be or not to be the person"));
spout.setCycle(true);
TridentTopology topology = new TridentTopology();
Stream stream = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"),
- new Split(), new Fields("word"))
- .window(TumblingCountWindow.of(1000), windowsStore, new Fields("word"), new CountAsAggregator(), new Fields("count"))
- .peek(new Consumer() {
- @Override
- public void accept(TridentTuple input) {
- LOG.info("Received tuple: [{}]", input);
- }
- });
+ new Split(), new Fields("word"))
+ .window(TumblingCountWindow.of(1000), windowsStore, new Fields("word"), new CountAsAggregator(),
+ new Fields("count"))
+ .peek(new Consumer() {
+ @Override
+ public void accept(TridentTuple input) {
+ LOG.info("Received tuple: [{}]", input);
+ }
+ });
return topology.build();
}
@@ -72,7 +67,8 @@ public class TridentHBaseWindowingStoreTopology {
conf.put(Config.TOPOLOGY_TRIDENT_WINDOWING_INMEMORY_CACHE_LIMIT, 100);
// window-state table should already be created with cf:tuples column
- HBaseWindowsStoreFactory windowStoreFactory = new HBaseWindowsStoreFactory(new HashMap<String, Object>(), "window-state", "cf".getBytes("UTF-8"), "tuples".getBytes("UTF-8"));
+ HBaseWindowsStoreFactory windowStoreFactory =
+ new HBaseWindowsStoreFactory(new HashMap<String, Object>(), "window-state", "cf".getBytes("UTF-8"), "tuples".getBytes("UTF-8"));
String topoName = "wordCounterWithWindowing";
if (args.length > 0) {
topoName = args[0];
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java
index c5f73ff..067eeff 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMapExample.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.trident;
import java.util.ArrayList;
import java.util.List;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -74,25 +68,25 @@ public class TridentMapExample {
public static StormTopology buildTopology() {
FixedBatchSpout spout = new FixedBatchSpout(
- new Fields("word"), 3, new Values("the cow jumped over the moon"),
- new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"),
- new Values("how many apples can you eat"), new Values("to be or not to be the person"));
+ new Fields("word"), 3, new Values("the cow jumped over the moon"),
+ new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"),
+ new Values("how many apples can you eat"), new Values("to be or not to be the person"));
spout.setCycle(true);
TridentTopology topology = new TridentTopology();
TridentState wordCounts = topology.newStream("spout1", spout).parallelismHint(16)
- .flatMap(split)
- .map(toUpper, new Fields("uppercased"))
- .filter(theFilter)
- .peek(new Consumer() {
- @Override
- public void accept(TridentTuple input) {
- System.out.println(input.getString(0));
- }
- })
- .groupBy(new Fields("uppercased"))
- .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"))
- .parallelismHint(16);
+ .flatMap(split)
+ .map(toUpper, new Fields("uppercased"))
+ .filter(theFilter)
+ .peek(new Consumer() {
+ @Override
+ public void accept(TridentTuple input) {
+ System.out.println(input.getString(0));
+ }
+ })
+ .groupBy(new Fields("uppercased"))
+ .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"))
+ .parallelismHint(16);
topology.newDRPCStream("words")
.flatMap(split, new Fields("word"))
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java
index 1cc33a9..bde5baa 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfDevicesTopology.java
@@ -1,27 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.trident;
import java.io.Serializable;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -55,13 +49,13 @@ public class TridentMinMaxOfDevicesTopology {
TridentTopology topology = new TridentTopology();
Stream devicesStream = topology.newStream("devicegen-spout", spout).
- each(allFields, new Debug("##### devices"));
+ each(allFields, new Debug("##### devices"));
devicesStream.minBy(deviceID).
- each(allFields, new Debug("#### device with min id"));
+ each(allFields, new Debug("#### device with min id"));
devicesStream.maxBy(count).
- each(allFields, new Debug("#### device with max count"));
+ each(allFields, new Debug("#### device with max count"));
return topology.build();
}
@@ -80,27 +74,27 @@ public class TridentMinMaxOfDevicesTopology {
TridentTopology topology = new TridentTopology();
Stream vehiclesStream = topology.newStream("spout1", spout).
- each(allFields, new Debug("##### vehicles"));
+ each(allFields, new Debug("##### vehicles"));
Stream slowVehiclesStream =
- vehiclesStream
- .min(new SpeedComparator())
- .each(vehicleField, new Debug("#### slowest vehicle"));
+ vehiclesStream
+ .min(new SpeedComparator())
+ .each(vehicleField, new Debug("#### slowest vehicle"));
Stream slowDriversStream =
- slowVehiclesStream
- .project(driverField)
- .each(driverField, new Debug("##### slowest driver"));
+ slowVehiclesStream
+ .project(driverField)
+ .each(driverField, new Debug("##### slowest driver"));
vehiclesStream
- .max(new SpeedComparator())
- .each(vehicleField, new Debug("#### fastest vehicle"))
- .project(driverField)
- .each(driverField, new Debug("##### fastest driver"));
+ .max(new SpeedComparator())
+ .each(vehicleField, new Debug("#### fastest vehicle"))
+ .project(driverField)
+ .each(driverField, new Debug("##### fastest driver"));
vehiclesStream
- .max(new EfficiencyComparator()).
- each(vehicleField, new Debug("#### efficient vehicle"));
+ .max(new EfficiencyComparator()).
+ each(vehicleField, new Debug("#### efficient vehicle"));
return topology.build();
}
@@ -148,9 +142,9 @@ public class TridentMinMaxOfDevicesTopology {
@Override
public String toString() {
return "Driver{" +
- "name='" + name + '\'' +
- ", id=" + id +
- '}';
+ "name='" + name + '\'' +
+ ", id=" + id +
+ '}';
}
}
@@ -166,26 +160,27 @@ public class TridentMinMaxOfDevicesTopology {
this.efficiency = efficiency;
}
- @Override
- public String toString() {
- return "Vehicle{" +
- "name='" + name + '\'' +
- ", maxSpeed=" + maxSpeed +
- ", efficiency=" + efficiency +
- '}';
- }
-
public static List<Object>[] generateVehicles(int count) {
List<Object>[] vehicles = new List[count];
for (int i = 0; i < count; i++) {
int id = i - 1;
vehicles[i] =
- (new Values(
- new Vehicle("Vehicle-" + id, ThreadLocalRandom.current().nextInt(0, 100), ThreadLocalRandom.current().nextDouble(1, 5)),
- new Driver("Driver-" + id, id)
- ));
+ (new Values(
+ new Vehicle("Vehicle-" + id, ThreadLocalRandom.current().nextInt(0, 100),
+ ThreadLocalRandom.current().nextDouble(1, 5)),
+ new Driver("Driver-" + id, id)
+ ));
}
return vehicles;
}
+
+ @Override
+ public String toString() {
+ return "Vehicle{" +
+ "name='" + name + '\'' +
+ ", maxSpeed=" + maxSpeed +
+ ", efficiency=" + efficiency +
+ '}';
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java
index 0d77914..f37442b 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentMinMaxOfVehiclesTopology.java
@@ -1,27 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.trident;
import java.io.Serializable;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -57,31 +51,31 @@ public class TridentMinMaxOfVehiclesTopology {
TridentTopology topology = new TridentTopology();
Stream vehiclesStream = topology.newStream("spout1", spout).
- each(allFields, new Debug("##### vehicles"));
+ each(allFields, new Debug("##### vehicles"));
Stream slowVehiclesStream =
- vehiclesStream
- .min(new SpeedComparator())
- .each(vehicleField, new Debug("#### slowest vehicle"));
+ vehiclesStream
+ .min(new SpeedComparator())
+ .each(vehicleField, new Debug("#### slowest vehicle"));
Stream slowDriversStream =
- slowVehiclesStream
- .project(driverField)
- .each(driverField, new Debug("##### slowest driver"));
+ slowVehiclesStream
+ .project(driverField)
+ .each(driverField, new Debug("##### slowest driver"));
vehiclesStream
- .max(new SpeedComparator())
- .each(vehicleField, new Debug("#### fastest vehicle"))
- .project(driverField)
- .each(driverField, new Debug("##### fastest driver"));
+ .max(new SpeedComparator())
+ .each(vehicleField, new Debug("#### fastest vehicle"))
+ .project(driverField)
+ .each(driverField, new Debug("##### fastest driver"));
vehiclesStream
- .minBy(Vehicle.FIELD_NAME, new EfficiencyComparator()).
- each(vehicleField, new Debug("#### least efficient vehicle"));
+ .minBy(Vehicle.FIELD_NAME, new EfficiencyComparator()).
+ each(vehicleField, new Debug("#### least efficient vehicle"));
vehiclesStream
- .maxBy(Vehicle.FIELD_NAME, new EfficiencyComparator()).
- each(vehicleField, new Debug("#### most efficient vehicle"));
+ .maxBy(Vehicle.FIELD_NAME, new EfficiencyComparator()).
+ each(vehicleField, new Debug("#### most efficient vehicle"));
return topology.build();
}
@@ -127,9 +121,9 @@ public class TridentMinMaxOfVehiclesTopology {
@Override
public String toString() {
return "Driver{" +
- "name='" + name + '\'' +
- ", id=" + id +
- '}';
+ "name='" + name + '\'' +
+ ", id=" + id +
+ '}';
}
}
@@ -145,26 +139,27 @@ public class TridentMinMaxOfVehiclesTopology {
this.efficiency = efficiency;
}
- @Override
- public String toString() {
- return "Vehicle{" +
- "name='" + name + '\'' +
- ", maxSpeed=" + maxSpeed +
- ", efficiency=" + efficiency +
- '}';
- }
-
public static List<Object>[] generateVehicles(int count) {
List<Object>[] vehicles = new List[count];
for (int i = 0; i < count; i++) {
int id = i - 1;
vehicles[i] =
- (new Values(
- new Vehicle("Vehicle-" + id, ThreadLocalRandom.current().nextInt(0, 100), ThreadLocalRandom.current().nextDouble(1, 5)),
- new Driver("Driver-" + id, id)
- ));
+ (new Values(
+ new Vehicle("Vehicle-" + id, ThreadLocalRandom.current().nextInt(0, 100),
+ ThreadLocalRandom.current().nextDouble(1, 5)),
+ new Driver("Driver-" + id, id)
+ ));
}
return vehicles;
}
+
+ @Override
+ public String toString() {
+ return "Vehicle{" +
+ "name='" + name + '\'' +
+ ", maxSpeed=" + maxSpeed +
+ ", efficiency=" + efficiency +
+ '}';
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java
index 8e5229d..19c0665 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentReach.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.trident;
import java.util.ArrayList;
@@ -22,7 +17,6 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.StormTopology;
@@ -60,28 +54,41 @@ public class TridentReach {
put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob"));
}};
- public static class StaticSingleKeyMapState extends ReadOnlyState implements ReadOnlyMapState<Object> {
- public static class Factory implements StateFactory {
- Map _map;
+ public static StormTopology buildTopology() {
+ TridentTopology topology = new TridentTopology();
+ TridentState urlToTweeters = topology.newStaticState(new StaticSingleKeyMapState.Factory(TWEETERS_DB));
+ TridentState tweetersToFollowers = topology.newStaticState(new StaticSingleKeyMapState.Factory(FOLLOWERS_DB));
- public Factory(Map map) {
- _map = map;
- }
- @Override
- public State makeState(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
- return new StaticSingleKeyMapState(_map);
- }
+ topology.newDRPCStream("reach").stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields(
+ "tweeters")).each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")).shuffle().stateQuery(
+ tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")).each(new Fields("followers"),
+ new ExpandList(),
+ new Fields("follower"))
+ .groupBy(new Fields("follower")).aggregate(new One(), new Fields(
+ "one")).aggregate(new Fields("one"), new Sum(), new Fields("reach"));
+ return topology.build();
+ }
+ public static void main(String[] args) throws Exception {
+ Config conf = new Config();
+ StormSubmitter.submitTopology("reach", conf, buildTopology());
+ try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
+ Thread.sleep(2000);
+
+ System.out.println("REACH: " + drpc.execute("reach", "aaa"));
+ System.out.println("REACH: " + drpc.execute("reach", "foo.com/blog/1"));
+ System.out.println("REACH: " + drpc.execute("reach", "engineering.twitter.com/blog/5"));
}
+ }
+ public static class StaticSingleKeyMapState extends ReadOnlyState implements ReadOnlyMapState<Object> {
Map _map;
public StaticSingleKeyMapState(Map map) {
_map = map;
}
-
@Override
public List<Object> multiGet(List<List<Object>> keys) {
List<Object> ret = new ArrayList();
@@ -92,6 +99,20 @@ public class TridentReach {
return ret;
}
+ public static class Factory implements StateFactory {
+ Map _map;
+
+ public Factory(Map map) {
+ _map = map;
+ }
+
+ @Override
+ public State makeState(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ return new StaticSingleKeyMapState(_map);
+ }
+
+ }
+
}
public static class One implements CombinerAggregator<Integer> {
@@ -124,30 +145,4 @@ public class TridentReach {
}
}
-
- public static StormTopology buildTopology() {
- TridentTopology topology = new TridentTopology();
- TridentState urlToTweeters = topology.newStaticState(new StaticSingleKeyMapState.Factory(TWEETERS_DB));
- TridentState tweetersToFollowers = topology.newStaticState(new StaticSingleKeyMapState.Factory(FOLLOWERS_DB));
-
-
- topology.newDRPCStream("reach").stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields(
- "tweeters")).each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")).shuffle().stateQuery(
- tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")).each(new Fields("followers"),
- new ExpandList(), new Fields("follower")).groupBy(new Fields("follower")).aggregate(new One(), new Fields(
- "one")).aggregate(new Fields("one"), new Sum(), new Fields("reach"));
- return topology.build();
- }
-
- public static void main(String[] args) throws Exception {
- Config conf = new Config();
- StormSubmitter.submitTopology("reach", conf, buildTopology());
- try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
- Thread.sleep(2000);
-
- System.out.println("REACH: " + drpc.execute("reach", "aaa"));
- System.out.println("REACH: " + drpc.execute("reach", "foo.com/blog/1"));
- System.out.println("REACH: " + drpc.execute("reach", "engineering.twitter.com/blog/5"));
- }
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java
index 2da29bf..862d09a 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWindowingInmemoryStoreTopology.java
@@ -1,21 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.trident;
import org.apache.storm.Config;
@@ -45,21 +39,22 @@ public class TridentWindowingInmemoryStoreTopology {
public static StormTopology buildTopology(WindowsStoreFactory windowStore, WindowConfig windowConfig) throws Exception {
FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"),
- new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"),
- new Values("how many apples can you eat"), new Values("to be or not to be the person"));
+ new Values("the man went to the store and bought some candy"),
+ new Values("four score and seven years ago"),
+ new Values("how many apples can you eat"), new Values("to be or not to be the person"));
spout.setCycle(true);
TridentTopology topology = new TridentTopology();
Stream stream = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"),
- new Split(), new Fields("word"))
- .window(windowConfig, windowStore, new Fields("word"), new CountAsAggregator(), new Fields("count"))
- .peek(new Consumer() {
- @Override
- public void accept(TridentTuple input) {
- LOG.info("Received tuple: [{}]", input);
- }
- });
+ new Split(), new Fields("word"))
+ .window(windowConfig, windowStore, new Fields("word"), new CountAsAggregator(), new Fields("count"))
+ .peek(new Consumer() {
+ @Override
+ public void accept(TridentTuple input) {
+ LOG.info("Received tuple: [{}]", input);
+ }
+ });
return topology.build();
}
@@ -71,7 +66,7 @@ public class TridentWindowingInmemoryStoreTopology {
if (args.length > 0) {
topoName = args[0];
}
-
+
conf.setNumWorkers(3);
StormSubmitter.submitTopologyWithProgressBar(topoName, conf, buildTopology(mapState, SlidingCountWindow.of(1000, 100)));
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java
index 0f86e1f..bafeba2 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/trident/TridentWordCount.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.trident;
import org.apache.storm.Config;
@@ -36,32 +31,25 @@ import org.apache.storm.utils.DRPCClient;
public class TridentWordCount {
- public static class Split extends BaseFunction {
- @Override
- public void execute(TridentTuple tuple, TridentCollector collector) {
- String sentence = tuple.getString(0);
- for (String word : sentence.split(" ")) {
- collector.emit(new Values(word));
- }
- }
- }
-
public static StormTopology buildTopology() {
FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"),
- new Values("the man went to the store and bought some candy"), new Values("four score and seven years ago"),
- new Values("how many apples can you eat"), new Values("to be or not to be the person"));
+ new Values("the man went to the store and bought some candy"),
+ new Values("four score and seven years ago"),
+ new Values("how many apples can you eat"), new Values("to be or not to be the person"));
spout.setCycle(true);
TridentTopology topology = new TridentTopology();
TridentState wordCounts = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"),
- new Split(), new Fields("word")).groupBy(new Fields("word")).persistentAggregate(new MemoryMapState.Factory(),
- new Count(), new Fields("count")).parallelismHint(16);
+ new Split(), new Fields("word"))
+ .groupBy(new Fields("word")).persistentAggregate(new MemoryMapState.Factory(),
+ new Count(), new Fields("count"))
+ .parallelismHint(16);
topology.newDRPCStream("words").each(new Fields("args"), new Split(), new Fields("word"))
- .groupBy(new Fields("word"))
- .stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count"))
- .each(new Fields("count"), new FilterNull())
- .project(new Fields("word", "count"));
+ .groupBy(new Fields("word"))
+ .stateQuery(wordCounts, new Fields("word"), new MapGet(), new Fields("count"))
+ .each(new Fields("count"), new FilterNull())
+ .project(new Fields("word", "count"));
return topology.build();
}
@@ -81,4 +69,14 @@ public class TridentWordCount {
}
}
}
+
+ public static class Split extends BaseFunction {
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ String sentence = tuple.getString(0);
+ for (String word : sentence.split(" ")) {
+ collector.emit(new Values(word));
+ }
+ }
+ }
}
[05/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java b/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
index 96ad939..a20cde0 100644
--- a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
@@ -91,9 +91,9 @@ public class ServerUtils {
private static ServerUtils _instance = new ServerUtils();
/**
- * Provide an instance of this class for delegates to use. To mock out
- * delegated methods, provide an instance of a subclass that overrides the
- * implementation of the delegated method.
+ * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that
+ * overrides the implementation of the delegated method.
+ *
* @param u a ServerUtils instance
* @return the previously set instance
*/
@@ -109,8 +109,8 @@ public class ServerUtils {
List<List<T>> rest = new ArrayList<List<T>>();
for (List<T> node : nodeList) {
if (node != null && node.size() > 0) {
- first.add(node.get(0));
- rest.add(node.subList(1, node.size()));
+ first.add(node.get(0));
+ rest.add(node.subList(1, node.size()));
}
}
List<T> interleaveRest = interleaveAll(rest);
@@ -120,14 +120,14 @@ public class ServerUtils {
return first;
}
return null;
- }
+ }
public static BlobStore getNimbusBlobStore(Map<String, Object> conf, NimbusInfo nimbusInfo) {
return getNimbusBlobStore(conf, null, nimbusInfo);
}
public static BlobStore getNimbusBlobStore(Map<String, Object> conf, String baseDir, NimbusInfo nimbusInfo) {
- String type = (String)conf.get(DaemonConfig.NIMBUS_BLOBSTORE);
+ String type = (String) conf.get(DaemonConfig.NIMBUS_BLOBSTORE);
if (type == null) {
type = LocalFsBlobStore.class.getName();
}
@@ -136,7 +136,7 @@ public class ServerUtils {
// only enable cleanup of blobstore on nimbus
nconf.put(Config.BLOBSTORE_CLEANUP_ENABLE, Boolean.TRUE);
- if(store != null) {
+ if (store != null) {
// store can be null during testing when mocking utils.
store.prepare(nconf, baseDir, nimbusInfo);
}
@@ -149,12 +149,13 @@ public class ServerUtils {
/**
* Returns the combined string, escaped for posix shell.
+ *
* @param command the list of strings to be combined
* @return the resulting command string
*/
- public static String shellCmd (List<String> command) {
+ public static String shellCmd(List<String> command) {
List<String> changedCommands = new ArrayList<>(command.size());
- for (String str: command) {
+ for (String str : command) {
if (str == null) {
continue;
}
@@ -164,29 +165,29 @@ public class ServerUtils {
}
/**
- * Takes an input dir or file and returns the disk usage on that local directory.
- * Very basic implementation.
+ * Takes an input dir or file and returns the disk usage on that local directory. Very basic implementation.
*
* @param dir The input dir to get the disk space of this local dir
* @return The total disk space of the input local directory
*/
public static long getDU(File dir) {
long size = 0;
- if (!dir.exists())
+ if (!dir.exists()) {
return 0;
+ }
if (!dir.isDirectory()) {
return dir.length();
} else {
File[] allFiles = dir.listFiles();
- if(allFiles != null) {
+ if (allFiles != null) {
for (int i = 0; i < allFiles.length; i++) {
boolean isSymLink;
try {
isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]);
- } catch(IOException ioe) {
+ } catch (IOException ioe) {
isSymLink = true;
}
- if(!isSymLink) {
+ if (!isSymLink) {
size += getDU(allFiles[i]);
}
}
@@ -209,6 +210,7 @@ public class ServerUtils {
/**
* Meant to be called only by the supervisor for stormjar/stormconf/stormcode files.
+ *
* @param key
* @param localFile
* @param cb
@@ -225,7 +227,7 @@ public class ServerUtils {
* Extract dir from the jar to destdir
*
* @param jarpath Path to the jar file
- * @param dir Directory in the jar to pull out
+ * @param dir Directory in the jar to pull out
* @param destdir Path to the directory where the extracted directory will be put
*/
public static void extractDirFromJar(String jarpath, String dir, File destdir) {
@@ -233,8 +235,8 @@ public class ServerUtils {
}
/**
- * Returns the value of java.class.path System property. Kept separate for
- * testing.
+ * Returns the value of java.class.path System property. Kept separate for testing.
+ *
* @return the classpath
*/
public static String currentClasspath() {
@@ -245,16 +247,16 @@ public class ServerUtils {
* Determines if a zip archive contains a particular directory.
*
* @param zipfile path to the zipped file
- * @param target directory being looked for in the zip.
+ * @param target directory being looked for in the zip.
* @return boolean whether or not the directory exists in the zip.
*/
public static boolean zipDoesContainDir(String zipfile, String target) throws IOException {
List<ZipEntry> entries = (List<ZipEntry>) Collections.list(new ZipFile(zipfile).entries());
String targetDir = target + "/";
- for(ZipEntry entry : entries) {
+ for (ZipEntry entry : entries) {
String name = entry.getName();
- if(name.startsWith(targetDir)) {
+ if (name.startsWith(targetDir)) {
return true;
}
}
@@ -266,23 +268,24 @@ public class ServerUtils {
return Files.getOwner(FileSystems.getDefault().getPath(path)).getName();
}
- public static String containerFilePath (String dir) {
+ public static String containerFilePath(String dir) {
return dir + File.separator + "launch_container.sh";
}
- public static String scriptFilePath (String dir) {
+ public static String scriptFilePath(String dir) {
return dir + File.separator + "storm-worker-script.sh";
}
/**
* Writes a posix shell script file to be executed in its own process.
- * @param dir the directory under which the script is to be written
- * @param command the command the script is to execute
+ *
+ * @param dir the directory under which the script is to be written
+ * @param command the command the script is to execute
* @param environment optional environment variables to set before running the script's command. May be null.
* @return the path to the script that has been written
*/
public static String writeScript(String dir, List<String> command,
- Map<String,String> environment) throws IOException {
+ Map<String, String> environment) throws IOException {
String path = scriptFilePath(dir);
try (BufferedWriter out = new BufferedWriter(new FileWriter(path))) {
out.write("#!/bin/bash");
@@ -294,14 +297,14 @@ public class ServerUtils {
v = "";
}
out.write(shellCmd(
- Arrays.asList(
- "export",k+"="+v)));
+ Arrays.asList(
+ "export", k + "=" + v)));
out.write(";");
out.newLine();
}
}
out.newLine();
- out.write("exec "+ shellCmd(command)+";");
+ out.write("exec " + shellCmd(command) + ";");
}
return path;
}
@@ -336,11 +339,11 @@ public class ServerUtils {
}
}
- public static void killProcessWithSigTerm (String pid) throws IOException {
+ public static void killProcessWithSigTerm(String pid) throws IOException {
sendSignalToProcess(Long.parseLong(pid), SIGTERM);
}
- public static void forceKillProcess (String pid) throws IOException {
+ public static void forceKillProcess(String pid) throws IOException {
sendSignalToProcess(Long.parseLong(pid), SIGKILL);
}
@@ -370,14 +373,13 @@ public class ServerUtils {
}
/**
- * Unpack matching files from a jar. Entries inside the jar that do
- * not match the given pattern will be skipped.
+ * Unpack matching files from a jar. Entries inside the jar that do not match the given pattern will be skipped.
*
* @param jarFile the .jar file to unpack
- * @param toDir the destination directory into which to unpack the jar
+ * @param toDir the destination directory into which to unpack the jar
*/
public static void unJar(File jarFile, File toDir)
- throws IOException {
+ throws IOException {
JarFile jar = new JarFile(jarFile);
try {
Enumeration<JarEntry> entries = jar.entries();
@@ -407,13 +409,13 @@ public class ServerUtils {
/**
* Copies from one stream to another.
*
- * @param in InputStream to read from
- * @param out OutputStream to write to
+ * @param in InputStream to read from
+ * @param out OutputStream to write to
* @param buffSize the size of the buffer
*/
public static void copyBytes(InputStream in, OutputStream out, int buffSize)
- throws IOException {
- PrintStream ps = out instanceof PrintStream ? (PrintStream)out : null;
+ throws IOException {
+ PrintStream ps = out instanceof PrintStream ? (PrintStream) out : null;
byte buf[] = new byte[buffSize];
int bytesRead = in.read(buf);
while (bytesRead >= 0) {
@@ -433,13 +435,12 @@ public class ServerUtils {
private static void ensureDirectory(File dir) throws IOException {
if (!dir.mkdirs() && !dir.isDirectory()) {
throw new IOException("Mkdirs failed to create " +
- dir.toString());
+ dir.toString());
}
}
/**
- * Given a Tar File as input it will untar the file in a the untar directory
- * passed as the second parameter
+ * Given a Tar File as input it will untar the file in a the untar directory passed as the second parameter
* <p/>
* This utility will untar ".tar" files and ".tar.gz","tgz" files.
*
@@ -484,13 +485,13 @@ public class ServerUtils {
} else {
untarCommand.append(inFile.toString());
}
- String[] shellCmd = {"bash", "-c", untarCommand.toString()};
+ String[] shellCmd = { "bash", "-c", untarCommand.toString() };
ShellUtils.ShellCommandExecutor shexec = new ShellUtils.ShellCommandExecutor(shellCmd);
shexec.execute();
int exitcode = shexec.getExitCode();
if (exitcode != 0) {
throw new IOException("Error untarring file " + inFile +
- ". Tar process exited with exit code " + exitcode);
+ ". Tar process exited with exit code " + exitcode);
}
}
@@ -500,7 +501,7 @@ public class ServerUtils {
try {
if (gzipped) {
inputStream = new BufferedInputStream(new GZIPInputStream(
- new FileInputStream(inFile)));
+ new FileInputStream(inFile)));
} else {
inputStream = new BufferedInputStream(new FileInputStream(inFile));
}
@@ -511,7 +512,7 @@ public class ServerUtils {
}
}
} finally {
- if(inputStream != null) {
+ if (inputStream != null) {
inputStream.close();
}
}
@@ -523,7 +524,7 @@ public class ServerUtils {
File subDir = new File(outputDir, entry.getName());
if (!subDir.mkdirs() && !subDir.isDirectory()) {
throw new IOException("Mkdirs failed to create tar internal dir "
- + outputDir);
+ + outputDir);
}
for (TarArchiveEntry e : entry.getDirectoryEntries()) {
unpackEntries(tis, e, subDir);
@@ -540,7 +541,7 @@ public class ServerUtils {
int count;
byte data[] = new byte[2048];
BufferedOutputStream outputStream = new BufferedOutputStream(
- new FileOutputStream(outputFile));
+ new FileOutputStream(outputFile));
while ((count = tis.read(data)) != -1) {
outputStream.write(data, 0, count);
@@ -556,14 +557,14 @@ public class ServerUtils {
} else if (lowerDst.endsWith(".zip")) {
unZip(localrsrc, dst);
} else if (lowerDst.endsWith(".tar.gz") ||
- lowerDst.endsWith(".tgz") ||
- lowerDst.endsWith(".tar")) {
+ lowerDst.endsWith(".tgz") ||
+ lowerDst.endsWith(".tar")) {
unTar(localrsrc, dst);
} else {
LOG.warn("Cannot unpack " + localrsrc);
if (!localrsrc.renameTo(dst)) {
throw new IOException("Unable to rename file: [" + localrsrc
- + "] to [" + dst + "]");
+ + "] to [" + dst + "]");
}
}
if (localrsrc.isFile()) {
@@ -572,9 +573,9 @@ public class ServerUtils {
}
/**
- * Given a File input it will unzip the file in a the unzip directory
- * passed as the second parameter
- * @param inFile The zip file as input
+ * Given a File input it will unzip the file in a the unzip directory passed as the second parameter
+ *
+ * @param inFile The zip file as input
* @param unzipDir The unzip directory where to unzip the zip file.
* @throws IOException
*/
@@ -617,14 +618,15 @@ public class ServerUtils {
}
/**
- * Given a zip File input it will return its size
- * Only works for zip files whose uncompressed size is less than 4 GB,
- * otherwise returns the size module 2^32, per gzip specifications
+ * Given a zip File input it will return its size Only works for zip files whose uncompressed size is less than 4 GB, otherwise returns
+ * the size module 2^32, per gzip specifications
+ *
* @param myFile The zip file as input
- * @throws IOException
* @return zip file size as a long
+ *
+ * @throws IOException
*/
- public static long zipFileSize(File myFile) throws IOException{
+ public static long zipFileSize(File myFile) throws IOException {
RandomAccessFile raf = new RandomAccessFile(myFile, "r");
raf.seek(raf.length() - 4);
long b4 = raf.read();
@@ -636,42 +638,6 @@ public class ServerUtils {
return val;
}
- // Non-static impl methods exist for mocking purposes.
- public String currentClasspathImpl() {
- return System.getProperty("java.class.path");
- }
-
- public void extractDirFromJarImpl(String jarpath, String dir, File destdir) {
- try (JarFile jarFile = new JarFile(jarpath)) {
- Enumeration<JarEntry> jarEnums = jarFile.entries();
- while (jarEnums.hasMoreElements()) {
- JarEntry entry = jarEnums.nextElement();
- if (!entry.isDirectory() && entry.getName().startsWith(dir)) {
- File aFile = new File(destdir, entry.getName());
- aFile.getParentFile().mkdirs();
- try (FileOutputStream out = new FileOutputStream(aFile);
- InputStream in = jarFile.getInputStream(entry)) {
- IOUtils.copy(in, out);
- }
- }
- }
- } catch (IOException e) {
- LOG.info("Could not extract {} from {}", dir, jarpath);
- }
- }
-
- public void downloadResourcesAsSupervisorImpl(String key, String localFile,
- ClientBlobStore cb) throws AuthorizationException, KeyNotFoundException, IOException {
- final int MAX_RETRY_ATTEMPTS = 2;
- final int ATTEMPTS_INTERVAL_TIME = 100;
- for (int retryAttempts = 0; retryAttempts < MAX_RETRY_ATTEMPTS; retryAttempts++) {
- if (downloadResourcesAsSupervisorAttempt(cb, key, localFile)) {
- break;
- }
- Utils.sleep(ATTEMPTS_INTERVAL_TIME);
- }
- }
-
private static boolean downloadResourcesAsSupervisorAttempt(ClientBlobStore cb, String key, String localFile) {
boolean isSuccess = false;
try (FileOutputStream out = new FileOutputStream(localFile);
@@ -702,6 +668,7 @@ public class ServerUtils {
/**
* Check if the scheduler is resource aware or not.
+ *
* @param conf The configuration
* @return True if it's resource aware; false otherwise
*/
@@ -717,7 +684,7 @@ public class ServerUtils {
public static int getEstimatedWorkerCountForRASTopo(Map<String, Object> topoConf, StormTopology topology)
throws InvalidTopologyException {
return (int) Math.ceil(getEstimatedTotalHeapMemoryRequiredByTopo(topoConf, topology) /
- ObjectReader.getDouble(topoConf.get(Config.WORKER_HEAP_MEMORY_MB)));
+ ObjectReader.getDouble(topoConf.get(Config.WORKER_HEAP_MEMORY_MB)));
}
public static double getEstimatedTotalHeapMemoryRequiredByTopo(Map<String, Object> topoConf, StormTopology topology)
@@ -725,13 +692,13 @@ public class ServerUtils {
Map<String, Integer> componentParallelism = getComponentParallelism(topoConf, topology);
double totalMemoryRequired = 0.0;
- for (Map.Entry<String, NormalizedResourceRequest> entry: ResourceUtils.getBoltsResources(topology, topoConf).entrySet()) {
+ for (Map.Entry<String, NormalizedResourceRequest> entry : ResourceUtils.getBoltsResources(topology, topoConf).entrySet()) {
int parallelism = componentParallelism.getOrDefault(entry.getKey(), 1);
double memoryRequirement = entry.getValue().getOnHeapMemoryMb();
totalMemoryRequired += memoryRequirement * parallelism;
}
- for (Map.Entry<String, NormalizedResourceRequest> entry: ResourceUtils.getSpoutsResources(topology, topoConf).entrySet()) {
+ for (Map.Entry<String, NormalizedResourceRequest> entry : ResourceUtils.getSpoutsResources(topology, topoConf).entrySet()) {
int parallelism = componentParallelism.getOrDefault(entry.getKey(), 1);
double memoryRequirement = entry.getValue().getOnHeapMemoryMb();
totalMemoryRequired += memoryRequirement * parallelism;
@@ -766,4 +733,40 @@ public class ServerUtils {
sub.getPrincipals().add(principal);
return sub;
}
+
+ // Non-static impl methods exist for mocking purposes.
+ public String currentClasspathImpl() {
+ return System.getProperty("java.class.path");
+ }
+
+ public void extractDirFromJarImpl(String jarpath, String dir, File destdir) {
+ try (JarFile jarFile = new JarFile(jarpath)) {
+ Enumeration<JarEntry> jarEnums = jarFile.entries();
+ while (jarEnums.hasMoreElements()) {
+ JarEntry entry = jarEnums.nextElement();
+ if (!entry.isDirectory() && entry.getName().startsWith(dir)) {
+ File aFile = new File(destdir, entry.getName());
+ aFile.getParentFile().mkdirs();
+ try (FileOutputStream out = new FileOutputStream(aFile);
+ InputStream in = jarFile.getInputStream(entry)) {
+ IOUtils.copy(in, out);
+ }
+ }
+ }
+ } catch (IOException e) {
+ LOG.info("Could not extract {} from {}", dir, jarpath);
+ }
+ }
+
+ public void downloadResourcesAsSupervisorImpl(String key, String localFile,
+ ClientBlobStore cb) throws AuthorizationException, KeyNotFoundException, IOException {
+ final int MAX_RETRY_ATTEMPTS = 2;
+ final int ATTEMPTS_INTERVAL_TIME = 100;
+ for (int retryAttempts = 0; retryAttempts < MAX_RETRY_ATTEMPTS; retryAttempts++) {
+ if (downloadResourcesAsSupervisorAttempt(cb, key, localFile)) {
+ break;
+ }
+ Utils.sleep(ATTEMPTS_INTERVAL_TIME);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/utils/StormCommonInstaller.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/utils/StormCommonInstaller.java b/storm-server/src/main/java/org/apache/storm/utils/StormCommonInstaller.java
index a1b5596..6f29552 100644
--- a/storm-server/src/main/java/org/apache/storm/utils/StormCommonInstaller.java
+++ b/storm-server/src/main/java/org/apache/storm/utils/StormCommonInstaller.java
@@ -1,19 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.utils;
import org.apache.storm.daemon.StormCommon;
@@ -37,7 +33,7 @@ public class StormCommonInstaller implements AutoCloseable {
public void close() throws Exception {
if (StormCommon.setInstance(_oldInstance) != _curInstance) {
throw new IllegalStateException(
- "Instances of this resource must be closed in reverse order of opening.");
+ "Instances of this resource must be closed in reverse order of opening.");
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java b/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java
index 96bce15..bf95a76 100644
--- a/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java
+++ b/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.zookeeper;
@@ -73,7 +67,7 @@ public class AclEnforcement {
List<ACL> drpcFullAcl = new ArrayList<>(2);
drpcFullAcl.add(superUserAcl);
- String drpcAclString = (String)conf.get(Config.STORM_ZOOKEEPER_DRPC_ACL);
+ String drpcAclString = (String) conf.get(Config.STORM_ZOOKEEPER_DRPC_ACL);
if (drpcAclString != null) {
Id drpcAclId = Utils.parseZkId(drpcAclString, Config.STORM_ZOOKEEPER_DRPC_ACL);
ACL drpcUserAcl = new ACL(ZooDefs.Perms.READ, drpcAclId);
@@ -85,7 +79,7 @@ public class AclEnforcement {
String stormRoot = (String) conf.get(Config.STORM_ZOOKEEPER_ROOT);
try (CuratorFramework zk = ClientZookeeper.mkClient(conf, zkServers, port, "",
- new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS)) {
+ new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS)) {
if (zk.checkExists().forPath(stormRoot) != null) {
//First off we want to verify that ROOT is good
verifyAclStrict(zk, superAcl, stormRoot, fixUp);
@@ -97,7 +91,7 @@ public class AclEnforcement {
// Now that the root is fine we can start to look at the other paths under it.
try (CuratorFramework zk = ClientZookeeper.mkClient(conf, zkServers, port, stormRoot,
- new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS)) {
+ new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS)) {
//Next verify that the blob store is correct before we start it up.
if (zk.checkExists().forPath(ClusterUtils.BLOBSTORE_SUBTREE) != null) {
verifyAclStrictRecursive(zk, superAcl, ClusterUtils.BLOBSTORE_SUBTREE, fixUp);
@@ -119,7 +113,7 @@ public class AclEnforcement {
try {
Subject nimbusSubject = new Subject();
nimbusSubject.getPrincipals().add(new NimbusPrincipal());
- for (String topoId: topoIds) {
+ for (String topoId : topoIds) {
try {
String blobKey = topoId + "-stormconf.ser";
Map<String, Object> topoConf = Utils.fromCompressedJsonConf(bs.readBlob(blobKey, nimbusSubject));
@@ -210,7 +204,7 @@ public class AclEnforcement {
}
private static void verifyParentWithTopoChildrenDeleteDead(CuratorFramework zk, ACL superUserAcl, String path,
- Map<String, Id> topoToZkCreds, boolean fixUp, int perms) throws Exception {
+ Map<String, Id> topoToZkCreds, boolean fixUp, int perms) throws Exception {
if (zk.checkExists().forPath(path) != null) {
verifyAclStrict(zk, Arrays.asList(superUserAcl), path, fixUp);
Set<String> possiblyBadIds = new HashSet<>();
@@ -228,7 +222,7 @@ public class AclEnforcement {
if (!possiblyBadIds.isEmpty()) {
//Lets reread the children in STORMS as the source of truth and see if a new one was created in the background
possiblyBadIds.removeAll(zk.getChildren().forPath(ClusterUtils.STORMS_SUBTREE));
- for (String topoId: possiblyBadIds) {
+ for (String topoId : possiblyBadIds) {
//Now we know for sure that this is a bad id
String childPath = path + ClusterUtils.ZK_SEPERATOR + topoId;
zk.delete().deletingChildrenIfNeeded().forPath(childPath);
@@ -238,12 +232,12 @@ public class AclEnforcement {
}
private static void verifyParentWithReadOnlyTopoChildrenDeleteDead(CuratorFramework zk, ACL superUserAcl, String path,
- Map<String, Id> topoToZkCreds, boolean fixUp) throws Exception {
+ Map<String, Id> topoToZkCreds, boolean fixUp) throws Exception {
verifyParentWithTopoChildrenDeleteDead(zk, superUserAcl, path, topoToZkCreds, fixUp, ZooDefs.Perms.READ);
}
private static void verifyParentWithReadWriteTopoChildrenDeleteDead(CuratorFramework zk, ACL superUserAcl, String path,
- Map<String, Id> topoToZkCreds, boolean fixUp) throws Exception {
+ Map<String, Id> topoToZkCreds, boolean fixUp) throws Exception {
verifyParentWithTopoChildrenDeleteDead(zk, superUserAcl, path, topoToZkCreds, fixUp, ZooDefs.Perms.ALL);
}
@@ -271,7 +265,7 @@ public class AclEnforcement {
private static void verifyAclStrictRecursive(CuratorFramework zk, List<ACL> strictAcl, String path, boolean fixUp) throws Exception {
verifyAclStrict(zk, strictAcl, path, fixUp);
- for (String child: zk.getChildren().forPath(path)) {
+ for (String child : zk.getChildren().forPath(path)) {
String newPath = path + ClusterUtils.ZK_SEPERATOR + child;
verifyAclStrictRecursive(zk, strictAcl, newPath, fixUp);
}
@@ -295,7 +289,7 @@ public class AclEnforcement {
private static boolean equivalent(List<ACL> a, List<ACL> b) {
if (a.size() == b.size()) {
- for (ACL aAcl: a) {
+ for (ACL aAcl : a) {
if (!b.contains(aAcl)) {
return false;
}
@@ -305,10 +299,10 @@ public class AclEnforcement {
return false;
}
- public static void main(String [] args) throws Exception {
+ public static void main(String[] args) throws Exception {
Map<String, Object> conf = Utils.readStormConfig();
boolean fixUp = false;
- for (String arg: args) {
+ for (String arg : args) {
String a = arg.toLowerCase();
if ("-f".equals(a) || "--fixup".equals(a)) {
fixUp = true;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java
index 59313a0..2f2bb1f 100644
--- a/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java
+++ b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java
@@ -1,22 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.zookeeper;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.leader.LeaderLatch;
import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
@@ -32,12 +32,6 @@ import org.apache.zookeeper.data.ACL;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
-
public class LeaderElectorImp implements ILeaderElector {
private static Logger LOG = LoggerFactory.getLogger(LeaderElectorImp.class);
private final Map<String, Object> conf;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java b/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java
index 0b401a5..c5497b5 100644
--- a/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java
+++ b/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java
@@ -15,9 +15,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.zookeeper;
+import java.io.File;
+import java.net.BindException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.leader.LeaderLatch;
@@ -36,27 +46,17 @@ import org.apache.zookeeper.server.ZooKeeperServer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.net.BindException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicReference;
-
public class Zookeeper {
- private static Logger LOG = LoggerFactory.getLogger(Zookeeper.class);
-
// A singleton instance allows us to mock delegated static methods in our
// tests by subclassing.
private static final Zookeeper INSTANCE = new Zookeeper();
+ private static Logger LOG = LoggerFactory.getLogger(Zookeeper.class);
private static Zookeeper _instance = INSTANCE;
/**
- * Provide an instance of this class for delegates to use. To mock out
- * delegated methods, provide an instance of a subclass that overrides the
- * implementation of the delegated method.
+ * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that
+ * overrides the implementation of the delegated method.
*
* @param u a Zookeeper instance
*/
@@ -65,9 +65,8 @@ public class Zookeeper {
}
/**
- * Resets the singleton instance to the default. This is helpful to reset
- * the class to its original functionality when mocking is no longer
- * desired.
+ * Resets the singleton instance to the default. This is helpful to reset the class to its original functionality when mocking is no
+ * longer desired.
*/
public static void resetInstance() {
_instance = INSTANCE;
@@ -137,30 +136,35 @@ public class Zookeeper {
/**
* Get master leader elector.
- * @param conf Config.
- * @param zkClient ZkClient, the client must have a default Config.STORM_ZOOKEEPER_ROOT as root path.
- * @param blobStore {@link BlobStore}
- * @param tc {@link TopoCache}
+ *
+ * @param conf Config.
+ * @param zkClient ZkClient, the client must have a default Config.STORM_ZOOKEEPER_ROOT as root path.
+ * @param blobStore {@link BlobStore}
+ * @param tc {@link TopoCache}
* @param clusterState {@link IStormClusterState}
- * @param acls ACLs
+ * @param acls ACLs
* @return Instance of {@link ILeaderElector}
+ *
* @throws UnknownHostException
*/
public static ILeaderElector zkLeaderElector(Map<String, Object> conf, CuratorFramework zkClient, BlobStore blobStore,
- final TopoCache tc, IStormClusterState clusterState, List<ACL> acls) throws UnknownHostException {
+ final TopoCache tc, IStormClusterState clusterState, List<ACL> acls) throws
+ UnknownHostException {
return _instance.zkLeaderElectorImpl(conf, zkClient, blobStore, tc, clusterState, acls);
}
protected ILeaderElector zkLeaderElectorImpl(Map<String, Object> conf, CuratorFramework zk, BlobStore blobStore,
- final TopoCache tc, IStormClusterState clusterState, List<ACL> acls) throws UnknownHostException {
+ final TopoCache tc, IStormClusterState clusterState, List<ACL> acls) throws
+ UnknownHostException {
List<String> servers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS);
String leaderLockPath = "/leader-lock";
String id = NimbusInfo.fromConf(conf).toHostPortString();
AtomicReference<LeaderLatch> leaderLatchAtomicReference = new AtomicReference<>(new LeaderLatch(zk, leaderLockPath, id));
AtomicReference<LeaderLatchListener> leaderLatchListenerAtomicReference =
- new AtomicReference<>(leaderLatchListenerImpl(new LeaderListenerCallback(conf, zk, leaderLatchAtomicReference.get(), blobStore, tc, clusterState, acls)));
+ new AtomicReference<>(leaderLatchListenerImpl(
+ new LeaderListenerCallback(conf, zk, leaderLatchAtomicReference.get(), blobStore, tc, clusterState, acls)));
return new LeaderElectorImp(conf, servers, zk, leaderLockPath, id, leaderLatchAtomicReference,
- leaderLatchListenerAtomicReference, blobStore, tc, clusterState, acls);
+ leaderLatchListenerAtomicReference, blobStore, tc, clusterState, acls);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/DaemonConfigTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/DaemonConfigTest.java b/storm-server/src/test/java/org/apache/storm/DaemonConfigTest.java
index 2900111..67e210d 100644
--- a/storm-server/src/test/java/org/apache/storm/DaemonConfigTest.java
+++ b/storm-server/src/test/java/org/apache/storm/DaemonConfigTest.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm;
import java.lang.reflect.InvocationTargetException;
@@ -23,7 +18,6 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
-
import org.apache.storm.validation.ConfigValidation;
import org.junit.Assert;
import org.junit.Test;
@@ -37,11 +31,11 @@ public class DaemonConfigTest {
passCases.add(null);
passCases.add("some string");
- String[] stuff = {"some", "string", "list"};
+ String[] stuff = { "some", "string", "list" };
passCases.add(Arrays.asList(stuff));
failCases.add(42);
- Integer[] wrongStuff = {1, 2, 3};
+ Integer[] wrongStuff = { 1, 2, 3 };
failCases.add(Arrays.asList(wrongStuff));
//worker.childopts validates
@@ -61,32 +55,38 @@ public class DaemonConfigTest {
}
@Test
- public void testNimbusChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException, InstantiationException, IllegalAccessException {
+ public void testNimbusChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException,
+ InstantiationException, IllegalAccessException {
stringOrStringListTest(DaemonConfig.NIMBUS_CHILDOPTS);
}
@Test
- public void testLogviewerChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException, InstantiationException, IllegalAccessException {
+ public void testLogviewerChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException,
+ InstantiationException, IllegalAccessException {
stringOrStringListTest(DaemonConfig.LOGVIEWER_CHILDOPTS);
}
@Test
- public void testUiChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException, InstantiationException, IllegalAccessException {
+ public void testUiChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException,
+ InstantiationException, IllegalAccessException {
stringOrStringListTest(DaemonConfig.UI_CHILDOPTS);
}
@Test
- public void testPacemakerChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException, InstantiationException, IllegalAccessException {
+ public void testPacemakerChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException,
+ InstantiationException, IllegalAccessException {
stringOrStringListTest(DaemonConfig.PACEMAKER_CHILDOPTS);
}
@Test
- public void testDrpcChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException, InstantiationException, IllegalAccessException {
+ public void testDrpcChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException,
+ InstantiationException, IllegalAccessException {
stringOrStringListTest(DaemonConfig.DRPC_CHILDOPTS);
}
@Test
- public void testSupervisorChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException, InstantiationException, IllegalAccessException {
+ public void testSupervisorChildoptsIsStringOrStringList() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException,
+ InstantiationException, IllegalAccessException {
stringOrStringListTest(DaemonConfig.SUPERVISOR_CHILDOPTS);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/LocalStateTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/LocalStateTest.java b/storm-server/src/test/java/org/apache/storm/LocalStateTest.java
index a4625a8..ade26ac 100644
--- a/storm-server/src/test/java/org/apache/storm/LocalStateTest.java
+++ b/storm-server/src/test/java/org/apache/storm/LocalStateTest.java
@@ -1,42 +1,34 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.storm.generated.GlobalStreamId;
import org.apache.storm.testing.TmpPath;
import org.apache.storm.utils.LocalState;
-import org.apache.thrift.TBase;
import org.junit.Assert;
import org.junit.Test;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
public class LocalStateTest {
@Test
- public void testLocalState() throws Exception{
- try (TmpPath dir1_tmp = new TmpPath(); TmpPath dir2_tmp = new TmpPath() ) {
+ public void testLocalState() throws Exception {
+ try (TmpPath dir1_tmp = new TmpPath(); TmpPath dir2_tmp = new TmpPath()) {
GlobalStreamId globalStreamId_a = new GlobalStreamId("a", "a");
GlobalStreamId globalStreamId_b = new GlobalStreamId("b", "b");
GlobalStreamId globalStreamId_c = new GlobalStreamId("c", "c");
@@ -69,8 +61,8 @@ public class LocalStateTest {
String dir = tmp_dir.getPath();
LocalState ls = new LocalState(dir);
GlobalStreamId gs_a = new GlobalStreamId("a", "a");
- FileOutputStream data = FileUtils.openOutputStream(new File(dir,"12345"));
- FileOutputStream version = FileUtils.openOutputStream(new File(dir,"12345.version"));
+ FileOutputStream data = FileUtils.openOutputStream(new File(dir, "12345"));
+ FileOutputStream version = FileUtils.openOutputStream(new File(dir, "12345.version"));
Assert.assertNull(ls.get("c"));
ls.put("a", gs_a);
Assert.assertEquals(gs_a, ls.get("a"));
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/MessagingTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/MessagingTest.java b/storm-server/src/test/java/org/apache/storm/MessagingTest.java
index b0b6625..18cfc37 100644
--- a/storm-server/src/test/java/org/apache/storm/MessagingTest.java
+++ b/storm-server/src/test/java/org/apache/storm/MessagingTest.java
@@ -1,47 +1,43 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.generated.StormTopology;
-import org.apache.storm.testing.TestWordSpout;
-import org.apache.storm.testing.TestGlobalCount;
+import org.apache.storm.testing.CompleteTopologyParam;
import org.apache.storm.testing.FixedTuple;
import org.apache.storm.testing.MockedSources;
-import org.apache.storm.testing.CompleteTopologyParam;
+import org.apache.storm.testing.TestGlobalCount;
+import org.apache.storm.testing.TestWordSpout;
import org.apache.storm.topology.TopologyBuilder;
-
import org.junit.Assert;
import org.junit.Test;
-import java.util.*;
-
public class MessagingTest {
@Test
public void testLocalTransport() throws Exception {
Config topoConf = new Config();
topoConf.put(Config.TOPOLOGY_WORKERS, 2);
- topoConf.put(Config.STORM_MESSAGING_TRANSPORT , "org.apache.storm.messaging.netty.Context");
+ topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context");
try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime()
- .withSupervisors(1).withPortsPerSupervisor(2)
- .withDaemonConf(topoConf).build()) {
+ .withSupervisors(1).withPortsPerSupervisor(2)
+ .withDaemonConf(topoConf).build()) {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("1", new TestWordSpout(true), 2);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/PacemakerTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/PacemakerTest.java b/storm-server/src/test/java/org/apache/storm/PacemakerTest.java
index 7a00f77..5a9c130 100644
--- a/storm-server/src/test/java/org/apache/storm/PacemakerTest.java
+++ b/storm-server/src/test/java/org/apache/storm/PacemakerTest.java
@@ -1,22 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.storm.generated.HBMessage;
import org.apache.storm.generated.HBMessageData;
import org.apache.storm.generated.HBPulse;
@@ -26,9 +24,6 @@ import org.apache.storm.utils.Utils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
public class PacemakerTest {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/TestCgroups.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/TestCgroups.java b/storm-server/src/test/java/org/apache/storm/TestCgroups.java
index c785dde..874bd13 100644
--- a/storm-server/src/test/java/org/apache/storm/TestCgroups.java
+++ b/storm-server/src/test/java/org/apache/storm/TestCgroups.java
@@ -1,40 +1,32 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm;
-import org.apache.storm.utils.Utils;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.apache.storm.container.cgroup.CgroupManager;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import org.apache.storm.container.cgroup.CgroupManager;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Unit tests for CGroups
@@ -52,7 +44,8 @@ public class TestCgroups {
Config config = new Config();
config.putAll(Utils.readDefaultConfig());
//We don't want to run the test is CGroups are not setup
- Assume.assumeTrue("Check if CGroups are setup", ((boolean) config.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE)) == true);
+ Assume
+ .assumeTrue("Check if CGroups are setup", ((boolean) config.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE)) == true);
Assert.assertTrue("Check if STORM_CGROUP_HIERARCHY_DIR exists", stormCgroupHierarchyExists(config));
Assert.assertTrue("Check if STORM_SUPERVISOR_CGROUP_ROOTDIR exists", stormCgroupSupervisorRootDirExists(config));
@@ -69,13 +62,14 @@ public class TestCgroups {
command.append(entry).append(" ");
}
String correctCommand1 = config.get(DaemonConfig.STORM_CGROUP_CGEXEC_CMD) + " -g memory,cpu:/"
- + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + workerId + " ";
+ + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + workerId + " ";
String correctCommand2 = config.get(DaemonConfig.STORM_CGROUP_CGEXEC_CMD) + " -g cpu,memory:/"
- + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + workerId + " ";
- Assert.assertTrue("Check if cgroup launch command is correct", command.toString().equals(correctCommand1) || command.toString().equals(correctCommand2));
+ + config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR) + "/" + workerId + " ";
+ Assert.assertTrue("Check if cgroup launch command is correct",
+ command.toString().equals(correctCommand1) || command.toString().equals(correctCommand2));
String pathToWorkerCgroupDir = ((String) config.get(Config.STORM_CGROUP_HIERARCHY_DIR))
- + "/" + ((String) config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR)) + "/" + workerId;
+ + "/" + ((String) config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR)) + "/" + workerId;
Assert.assertTrue("Check if cgroup directory exists for worker", dirExists(pathToWorkerCgroupDir));
@@ -90,7 +84,8 @@ public class TestCgroups {
String pathTomemoryLimitInBytes = pathToWorkerCgroupDir + "/memory.limit_in_bytes";
Assert.assertTrue("Check if memory.limit_in_bytes file exists", fileExists(pathTomemoryLimitInBytes));
- Assert.assertEquals("Check if the correct value is written into memory.limit_in_bytes", String.valueOf(1024 * 1024 * 1024), readFileAll(pathTomemoryLimitInBytes));
+ Assert.assertEquals("Check if the correct value is written into memory.limit_in_bytes", String.valueOf(1024 * 1024 * 1024),
+ readFileAll(pathTomemoryLimitInBytes));
manager.releaseResourcesForWorker(workerId);
@@ -104,7 +99,7 @@ public class TestCgroups {
private boolean stormCgroupSupervisorRootDirExists(Map<String, Object> config) {
String pathTostormCgroupSupervisorRootDir = ((String) config.get(Config.STORM_CGROUP_HIERARCHY_DIR))
- + "/" + ((String) config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR));
+ + "/" + ((String) config.get(DaemonConfig.STORM_SUPERVISOR_CGROUP_ROOTDIR));
return dirExists(pathTostormCgroupSupervisorRootDir);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/TestDaemonConfigValidate.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/TestDaemonConfigValidate.java b/storm-server/src/test/java/org/apache/storm/TestDaemonConfigValidate.java
index fd00ee9..a3e01bf 100644
--- a/storm-server/src/test/java/org/apache/storm/TestDaemonConfigValidate.java
+++ b/storm-server/src/test/java/org/apache/storm/TestDaemonConfigValidate.java
@@ -18,21 +18,21 @@
package org.apache.storm;
-import org.apache.storm.validation.ConfigValidation;
-import org.junit.Assert;
-import org.junit.Test;
-
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
+import org.apache.storm.validation.ConfigValidation;
+import org.junit.Assert;
+import org.junit.Test;
public class TestDaemonConfigValidate {
@Test
- public void testSupervisorSchedulerMetaIsStringMap() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException, InstantiationException, IllegalAccessException {
+ public void testSupervisorSchedulerMetaIsStringMap() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException,
+ InstantiationException, IllegalAccessException {
Map<String, Object> conf = new HashMap<String, Object>();
Map<String, Object> schedulerMeta = new HashMap<String, Object>();
conf.put(DaemonConfig.SUPERVISOR_SCHEDULER_META, schedulerMeta);
@@ -52,7 +52,8 @@ public class TestDaemonConfigValidate {
}
@Test
- public void testIsolationSchedulerMachinesIsMap() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException, InstantiationException, IllegalAccessException {
+ public void testIsolationSchedulerMachinesIsMap() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException,
+ InstantiationException, IllegalAccessException {
Map<String, Object> conf = new HashMap<String, Object>();
Map<String, Integer> isolationMap = new HashMap<String, Integer>();
conf.put(DaemonConfig.ISOLATION_SCHEDULER_MACHINES, isolationMap);
@@ -73,19 +74,20 @@ public class TestDaemonConfigValidate {
}
@Test
- public void testSupervisorSlotsPorts() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException, InstantiationException, IllegalAccessException {
+ public void testSupervisorSlotsPorts() throws InvocationTargetException, NoSuchMethodException, NoSuchFieldException,
+ InstantiationException, IllegalAccessException {
Map<String, Object> conf = new HashMap<String, Object>();
Collection<Object> passCases = new LinkedList<Object>();
Collection<Object> failCases = new LinkedList<Object>();
- Integer[] test1 = {1233, 1234, 1235};
- Integer[] test2 = {1233};
+ Integer[] test1 = { 1233, 1234, 1235 };
+ Integer[] test2 = { 1233 };
passCases.add(Arrays.asList(test1));
passCases.add(Arrays.asList(test2));
- String[] test3 = {"1233", "1234", "1235"};
+ String[] test3 = { "1233", "1234", "1235" };
//duplicate case
- Integer[] test4 = {1233, 1233, 1235};
+ Integer[] test4 = { 1233, 1233, 1235 };
failCases.add(test3);
failCases.add(test4);
failCases.add(null);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/TestRebalance.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/TestRebalance.java b/storm-server/src/test/java/org/apache/storm/TestRebalance.java
index 912ad42..6f9f076 100644
--- a/storm-server/src/test/java/org/apache/storm/TestRebalance.java
+++ b/storm-server/src/test/java/org/apache/storm/TestRebalance.java
@@ -1,23 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.storm.generated.ClusterSummary;
import org.apache.storm.generated.RebalanceOptions;
import org.apache.storm.generated.StormTopology;
@@ -37,9 +33,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.Map;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -49,6 +42,15 @@ public class TestRebalance {
private static final Logger LOG = LoggerFactory.getLogger(TestRebalance.class);
+ public static String topoNameToId(String topoName, ILocalCluster cluster) throws TException {
+ for (TopologySummary topoSum : cluster.getClusterInfo().get_topologies()) {
+ if (topoSum.get_name().equals(topoName)) {
+ return topoSum.get_id();
+ }
+ }
+ return null;
+ }
+
@Test
public void testRebalanceTopologyResourcesAndConfigs()
throws Exception {
@@ -71,11 +73,11 @@ public class TestRebalance {
TopologyBuilder builder = new TopologyBuilder();
SpoutDeclarer s1 = builder.setSpout("spout-1", new TestUtilsForResourceAwareScheduler.TestSpout(),
- 2);
+ 2);
BoltDeclarer b1 = builder.setBolt("bolt-1", new TestUtilsForResourceAwareScheduler.TestBolt(),
- 2).shuffleGrouping("spout-1");
+ 2).shuffleGrouping("spout-1");
BoltDeclarer b2 = builder.setBolt("bolt-2", new TestUtilsForResourceAwareScheduler.TestBolt(),
- 2).shuffleGrouping("bolt-1");
+ 2).shuffleGrouping("bolt-1");
StormTopology stormTopology = builder.createTopology();
@@ -173,13 +175,4 @@ public class TestRebalance {
}
return false;
}
-
- public static String topoNameToId(String topoName, ILocalCluster cluster) throws TException {
- for (TopologySummary topoSum : cluster.getClusterInfo().get_topologies()) {
- if (topoSum.get_name().equals(topoName)) {
- return topoSum.get_id();
- }
- }
- return null;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/TestingTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/TestingTest.java b/storm-server/src/test/java/org/apache/storm/TestingTest.java
index 201b470..ee0fabc 100644
--- a/storm-server/src/test/java/org/apache/storm/TestingTest.java
+++ b/storm-server/src/test/java/org/apache/storm/TestingTest.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm;
@@ -37,7 +31,7 @@ import org.apache.storm.tuple.Values;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertTrue;
/**
* Test that the testing class does what it should do.
@@ -48,18 +42,18 @@ public class TestingTest {
TopologyBuilder tb = new TopologyBuilder();
tb.setSpout("spout", new TestWordSpout(true), 3);
tb.setBolt("2", new TestWordCounter(), 4)
- .fieldsGrouping("spout", new Fields("word"));
+ .fieldsGrouping("spout", new Fields("word"));
tb.setBolt("3", new TestGlobalCount())
- .globalGrouping("spout");
+ .globalGrouping("spout");
tb.setBolt("4", new TestAggregatesCounter())
- .globalGrouping("2");
+ .globalGrouping("2");
MockedSources mocked = new MockedSources();
mocked.addMockData("spout",
- new Values("nathan"),
- new Values("bob"),
- new Values("joey"),
- new Values("nathan"));
+ new Values("nathan"),
+ new Values("bob"),
+ new Values("joey"),
+ new Values("nathan"));
Config topoConf = new Config();
topoConf.setNumWorkers(2);
@@ -71,27 +65,27 @@ public class TestingTest {
Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, tb.createTopology(), ctp);
List<List<Object>> spoutTuples = Testing.readTuples(results, "spout");
List<List<Object>> expectedSpoutTuples = Arrays.asList(Arrays.asList("nathan"), Arrays.asList("bob"), Arrays.asList("joey"),
- Arrays.asList("nathan"));
+ Arrays.asList("nathan"));
assertTrue(expectedSpoutTuples + " expected, but found " + spoutTuples,
- Testing.multiseteq(expectedSpoutTuples, spoutTuples));
+ Testing.multiseteq(expectedSpoutTuples, spoutTuples));
List<List<Object>> twoTuples = Testing.readTuples(results, "2");
List<List<Object>> expectedTwoTuples = Arrays.asList(Arrays.asList("nathan", 1), Arrays.asList("nathan", 2),
- Arrays.asList("bob", 1), Arrays.asList("joey", 1));
+ Arrays.asList("bob", 1), Arrays.asList("joey", 1));
assertTrue(expectedTwoTuples + " expected, but found " + twoTuples,
- Testing.multiseteq(expectedTwoTuples, twoTuples));
+ Testing.multiseteq(expectedTwoTuples, twoTuples));
List<List<Object>> threeTuples = Testing.readTuples(results, "3");
List<List<Object>> expectedThreeTuples = Arrays.asList(Arrays.asList(1), Arrays.asList(2),
- Arrays.asList(3), Arrays.asList(4));
+ Arrays.asList(3), Arrays.asList(4));
assertTrue(expectedThreeTuples + " expected, but found " + threeTuples,
- Testing.multiseteq(expectedThreeTuples, threeTuples));
+ Testing.multiseteq(expectedThreeTuples, threeTuples));
List<List<Object>> fourTuples = Testing.readTuples(results, "4");
List<List<Object>> expectedFourTuples = Arrays.asList(Arrays.asList(1), Arrays.asList(2),
- Arrays.asList(3), Arrays.asList(4));
+ Arrays.asList(3), Arrays.asList(4));
assertTrue(expectedFourTuples + " expected, but found " + fourTuples,
- Testing.multiseteq(expectedFourTuples, fourTuples));
+ Testing.multiseteq(expectedFourTuples, fourTuples));
};
@Test
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/TickTupleTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/TickTupleTest.java b/storm-server/src/test/java/org/apache/storm/TickTupleTest.java
index 7f9f2b3..fdbcff5 100644
--- a/storm-server/src/test/java/org/apache/storm/TickTupleTest.java
+++ b/storm-server/src/test/java/org/apache/storm/TickTupleTest.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm;
@@ -39,7 +33,8 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
public class TickTupleTest {
private final static Logger LOG = LoggerFactory.getLogger(TickTupleTest.class);
@@ -48,11 +43,11 @@ public class TickTupleTest {
@Test
public void testTickTupleWorksWithSystemBolt() throws Exception {
- try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().build()){
+ try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().build()) {
StormTopology topology = createNoOpTopology();
Config topoConf = new Config();
topoConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 1);
- try (ILocalTopology topo = cluster.submitTopology("test", topoConf, topology)) {
+ try (ILocalTopology topo = cluster.submitTopology("test", topoConf, topology)) {
//Give the topology some time to come up
long time = 0;
int timeout = Math.max(Testing.TEST_TIMEOUT_MS, 100_000);
@@ -65,13 +60,20 @@ public class TickTupleTest {
for (int i = 0; i < 5; i++) {
cluster.advanceClusterTime(1);
time += 1_000;
- assertEquals("Iteration " + i, (Long)time, tickTupleTimes.poll(100, TimeUnit.MILLISECONDS));
+ assertEquals("Iteration " + i, (Long) time, tickTupleTimes.poll(100, TimeUnit.MILLISECONDS));
}
}
assertNull("The bolt got a tuple that is not a tick tuple " + nonTickTuple.get(), nonTickTuple.get());
}
}
+ private StormTopology createNoOpTopology() {
+ TopologyBuilder builder = new TopologyBuilder();
+ builder.setSpout("Spout", new NoopSpout());
+ builder.setBolt("Bolt", new NoopBolt()).fieldsGrouping("Spout", new Fields("tuple"));
+ return builder.createTopology();
+ }
+
private static class NoopSpout extends BaseRichSpout {
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
@@ -111,11 +113,4 @@ public class TickTupleTest {
@Override
public void declareOutputFields(OutputFieldsDeclarer ofd) {}
}
-
- private StormTopology createNoOpTopology() {
- TopologyBuilder builder = new TopologyBuilder();
- builder.setSpout("Spout", new NoopSpout());
- builder.setBolt("Bolt", new NoopBolt()).fieldsGrouping("Spout", new Fields("tuple"));
- return builder.createTopology();
- }
}
[21/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-core
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/planner/TaskBundle.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/planner/TaskBundle.java b/storm-core/src/jvm/org/apache/storm/planner/TaskBundle.java
index 2437c93..6052b6d 100644
--- a/storm-core/src/jvm/org/apache/storm/planner/TaskBundle.java
+++ b/storm-core/src/jvm/org/apache/storm/planner/TaskBundle.java
@@ -1,33 +1,28 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.planner;
-import org.apache.storm.task.IBolt;
import java.io.Serializable;
+import org.apache.storm.task.IBolt;
public class TaskBundle implements Serializable {
public IBolt task;
public int componentId;
-
+
public TaskBundle(IBolt task, int componentId) {
this.task = task;
this.componentId = componentId;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java b/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java
index 4710f10..4b0404f 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/MockLeaderElector.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.testing;
@@ -21,14 +15,13 @@ package org.apache.storm.testing;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.nimbus.ILeaderElector;
import org.apache.storm.nimbus.NimbusInfo;
public class MockLeaderElector implements ILeaderElector {
private final boolean isLeader;
private final NimbusInfo leaderAddress;
-
+
public MockLeaderElector() {
this(true, "test-host", 9999);
}
@@ -36,7 +29,7 @@ public class MockLeaderElector implements ILeaderElector {
public MockLeaderElector(boolean isLeader) {
this(isLeader, "test-host", 9999);
}
-
+
public MockLeaderElector(boolean isLeader, String host, int port) {
this.isLeader = isLeader;
this.leaderAddress = new NimbusInfo(host, port, true);
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedClientZookeeper.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedClientZookeeper.java b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedClientZookeeper.java
index b1f0b49..6c52134 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedClientZookeeper.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedClientZookeeper.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.testing.staticmocking;
import org.apache.storm.zookeeper.ClientZookeeper;
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedCluster.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedCluster.java b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedCluster.java
index 2f1440c..34bfbc3 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedCluster.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedCluster.java
@@ -1,24 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.testing.staticmocking;
import org.apache.storm.cluster.ClusterUtils;
-public class MockedCluster implements AutoCloseable {
+public class MockedCluster implements AutoCloseable {
public MockedCluster(ClusterUtils inst) {
ClusterUtils.setInstance(inst);
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java
index d33dc9c..f806bf7 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java
@@ -1,19 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.testing.staticmocking;
import org.apache.storm.daemon.supervisor.SupervisorUtils;
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedZookeeper.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedZookeeper.java b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedZookeeper.java
index ca54f6f..6207123 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedZookeeper.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedZookeeper.java
@@ -1,19 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.testing.staticmocking;
import org.apache.storm.zookeeper.Zookeeper;
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/ui/FilterConfiguration.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/ui/FilterConfiguration.java b/storm-core/src/jvm/org/apache/storm/ui/FilterConfiguration.java
index e671264..153826e 100644
--- a/storm-core/src/jvm/org/apache/storm/ui/FilterConfiguration.java
+++ b/storm-core/src/jvm/org/apache/storm/ui/FilterConfiguration.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.ui;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/ui/IConfigurator.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/ui/IConfigurator.java b/storm-core/src/jvm/org/apache/storm/ui/IConfigurator.java
index 86e47d4..9424677 100644
--- a/storm-core/src/jvm/org/apache/storm/ui/IConfigurator.java
+++ b/storm-core/src/jvm/org/apache/storm/ui/IConfigurator.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.ui;
import org.eclipse.jetty.server.Server;
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/ui/InvalidRequestException.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/ui/InvalidRequestException.java b/storm-core/src/jvm/org/apache/storm/ui/InvalidRequestException.java
index 1fb1d69..04f0596 100644
--- a/storm-core/src/jvm/org/apache/storm/ui/InvalidRequestException.java
+++ b/storm-core/src/jvm/org/apache/storm/ui/InvalidRequestException.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.ui;
public class InvalidRequestException extends Exception {
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java b/storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java
index be4a4c0..af8999b 100644
--- a/storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java
+++ b/storm-core/src/jvm/org/apache/storm/ui/UIHelpers.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.ui;
import com.google.common.base.Joiner;
@@ -50,17 +45,19 @@ import org.json.simple.JSONValue;
public class UIHelpers {
private static final Object[][] PRETTY_SEC_DIVIDERS = {
- new Object[]{"s", 60},
- new Object[]{"m", 60},
- new Object[]{"h", 24},
- new Object[]{"d", null}};
+ new Object[]{ "s", 60 },
+ new Object[]{ "m", 60 },
+ new Object[]{ "h", 24 },
+ new Object[]{ "d", null }
+ };
private static final Object[][] PRETTY_MS_DIVIDERS = {
- new Object[]{"ms", 1000},
- new Object[]{"s", 60},
- new Object[]{"m", 60},
- new Object[]{"h", 24},
- new Object[]{"d", null}};
+ new Object[]{ "ms", 1000 },
+ new Object[]{ "s", 60 },
+ new Object[]{ "m", 60 },
+ new Object[]{ "h", 24 },
+ new Object[]{ "d", null }
+ };
public static String prettyUptimeStr(String val, Object[][] dividers) {
int uptime = Integer.parseInt(val);
@@ -111,13 +108,13 @@ public class UIHelpers {
public static Map<String, Object> unauthorizedUserJson(String user) {
return ImmutableMap.of(
- "error", "No Authorization",
- "errorMessage", String.format("User %s is not authorized.", user));
+ "error", "No Authorization",
+ "errorMessage", String.format("User %s is not authorized.", user));
}
private static ServerConnector mkSslConnector(Server server, Integer port, String ksPath, String ksPassword, String ksType,
- String keyPassword, String tsPath, String tsPassword, String tsType,
- Boolean needClientAuth, Boolean wantClientAuth, Integer headerBufferSize) {
+ String keyPassword, String tsPath, String tsPassword, String tsType,
+ Boolean needClientAuth, Boolean wantClientAuth, Integer headerBufferSize) {
SslContextFactory factory = new SslContextFactory();
factory.setExcludeCipherSuites("SSL_RSA_WITH_RC4_128_MD5", "SSL_RSA_WITH_RC4_128_SHA");
factory.setExcludeProtocols("SSLv3");
@@ -145,8 +142,8 @@ public class UIHelpers {
httpsConfig.setRequestHeaderSize(headerBufferSize);
}
ServerConnector sslConnector = new ServerConnector(server,
- new SslConnectionFactory(factory, HttpVersion.HTTP_1_1.asString()),
- new HttpConnectionFactory(httpsConfig));
+ new SslConnectionFactory(factory, HttpVersion.HTTP_1_1.asString()),
+ new HttpConnectionFactory(httpsConfig));
sslConnector.setPort(port);
return sslConnector;
}
@@ -155,7 +152,7 @@ public class UIHelpers {
String keyPassword, String tsPath, String tsPassword, String tsType,
Boolean needClientAuth, Boolean wantClientAuth) {
configSsl(server, port, ksPath, ksPassword, ksType, keyPassword,
- tsPath, tsPassword, tsType, needClientAuth, wantClientAuth, null);
+ tsPath, tsPassword, tsType, needClientAuth, wantClientAuth, null);
}
public static void configSsl(Server server, Integer port, String ksPath, String ksPassword, String ksType,
@@ -163,7 +160,7 @@ public class UIHelpers {
Boolean needClientAuth, Boolean wantClientAuth, Integer headerBufferSize) {
if (port > 0) {
server.addConnector(mkSslConnector(server, port, ksPath, ksPassword, ksType, keyPassword,
- tsPath, tsPassword, tsType, needClientAuth, wantClientAuth, headerBufferSize));
+ tsPath, tsPassword, tsType, needClientAuth, wantClientAuth, headerBufferSize));
}
}
@@ -171,7 +168,9 @@ public class UIHelpers {
FilterHolder filterHolder = new FilterHolder(new CrossOriginFilter());
filterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
filterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "GET, POST, PUT");
- filterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "X-Requested-With, X-Requested-By, Access-Control-Allow-Origin, Content-Type, Content-Length, Accept, Origin");
+ filterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM,
+ "X-Requested-With, X-Requested-By, Access-Control-Allow-Origin, Content-Type, Content-Length, " +
+ "Accept, Origin");
filterHolder.setInitParameter(CrossOriginFilter.ACCESS_CONTROL_ALLOW_ORIGIN_HEADER, "*");
return filterHolder;
}
@@ -183,7 +182,7 @@ public class UIHelpers {
public static void configFilter(Server server, Servlet servlet, List<FilterConfiguration> filtersConfs) {
configFilter(server, servlet, filtersConfs, null);
}
-
+
public static void configFilter(Server server, Servlet servlet, List<FilterConfiguration> filtersConfs, Map<String, String> params) {
if (filtersConfs != null) {
ServletHolder servletHolder = new ServletHolder(servlet);
@@ -197,7 +196,7 @@ public class UIHelpers {
server.setHandler(context);
}
}
-
+
public static void configFilters(ServletContextHandler context, List<FilterConfiguration> filtersConfs) {
context.addFilter(corsFilterHandle(), "/*", EnumSet.allOf(DispatcherType.class));
for (FilterConfiguration filterConf : filtersConfs) {
@@ -256,7 +255,8 @@ public class UIHelpers {
* Modified version of run-jetty
* Assumes configurator sets handler.
*/
- public static void stormRunJetty(Integer port, String host, Integer httpsPort, Integer headerBufferSize, IConfigurator configurator) throws Exception {
+ public static void stormRunJetty(Integer port, String host, Integer httpsPort, Integer headerBufferSize,
+ IConfigurator configurator) throws Exception {
Server s = jettyCreateServer(port, host, httpsPort, headerBufferSize);
if (configurator != null) {
configurator.execute(s);
@@ -276,7 +276,9 @@ public class UIHelpers {
Map<String, String> headersResult = new HashMap<>();
headersResult.put("Cache-Control", "no-cache, no-store");
headersResult.put("Access-Control-Allow-Origin", "*");
- headersResult.put("Access-Control-Allow-Headers", "Content-Type, Access-Control-Allow-Headers, Access-Controler-Allow-Origin, X-Requested-By, X-Csrf-Token, Authorization, X-Requested-With");
+ headersResult.put("Access-Control-Allow-Headers",
+ "Content-Type, Access-Control-Allow-Headers, Access-Controler-Allow-Origin, X-Requested-By, X-Csrf-Token, " +
+ "Authorization, X-Requested-With");
if (callback != null) {
headersResult.put("Content-Type", "application/javascript;charset=utf-8");
} else {
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/utils/ClojureTimerTask.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/ClojureTimerTask.java b/storm-core/src/jvm/org/apache/storm/utils/ClojureTimerTask.java
index caefdb9..a220a23 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/ClojureTimerTask.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/ClojureTimerTask.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.utils;
import clojure.lang.IFn;
@@ -22,14 +17,14 @@ import java.util.TimerTask;
public class ClojureTimerTask extends TimerTask {
IFn _afn;
-
+
public ClojureTimerTask(IFn afn) {
super();
_afn = afn;
}
-
+
@Override
public void run() {
_afn.run();
- }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/utils/Monitor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/Monitor.java b/storm-core/src/jvm/org/apache/storm/utils/Monitor.java
index 15569d0..8e3ce27 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/Monitor.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/Monitor.java
@@ -1,26 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.utils;
-import org.apache.storm.generated.*;
+package org.apache.storm.utils;
import java.util.HashSet;
import java.util.Map;
+import org.apache.storm.generated.ClusterSummary;
+import org.apache.storm.generated.ExecutorStats;
+import org.apache.storm.generated.ExecutorSummary;
+import org.apache.storm.generated.GetInfoOptions;
+import org.apache.storm.generated.Nimbus;
+import org.apache.storm.generated.NumErrorsChoice;
+import org.apache.storm.generated.TopologyInfo;
+import org.apache.storm.generated.TopologySummary;
public class Monitor {
private static final String WATCH_TRANSFERRED = "transferred";
@@ -32,75 +33,11 @@ public class Monitor {
private String _stream;
private String _watch;
- private static class MetricsState {
- private long lastTime = 0;
- private long lastStatted = 0;
-
- private MetricsState(long lastTime, long lastStatted) {
- this.lastTime = lastTime;
- this.lastStatted = lastStatted;
- }
-
- public long getLastStatted() {
- return lastStatted;
- }
-
- public void setLastStatted(long lastStatted) {
- this.lastStatted = lastStatted;
- }
-
- public long getLastTime() {
- return lastTime;
- }
-
- public void setLastTime(long lastTime) {
- this.lastTime = lastTime;
- }
- }
-
- private static class Poller {
- private long startTime = 0;
- private long pollMs = 0;
-
- private Poller(long startTime, long pollMs) {
- this.startTime = startTime;
- this.pollMs = pollMs;
- }
-
- public long nextPoll() throws InterruptedException {
- long now = System.currentTimeMillis();
- long cycle = (now - startTime) / pollMs;
- long wakeupTime = startTime + (pollMs * (cycle + 1));
- long sleepTime = wakeupTime - now;
- if (sleepTime > 0) {
- Thread.sleep(sleepTime);
- }
- now = System.currentTimeMillis();
- return now;
- }
-
- public long getStartTime() {
- return startTime;
- }
-
- public void setStartTime(long startTime) {
- this.startTime = startTime;
- }
-
- public long getPollMs() {
- return pollMs;
- }
-
- public void setPollMs(long pollMs) {
- this.pollMs = pollMs;
- }
- }
-
- private HashSet<String> getComponents(Nimbus.Iface client, String topology) throws Exception{
+ private HashSet<String> getComponents(Nimbus.Iface client, String topology) throws Exception {
HashSet<String> components = new HashSet<>();
ClusterSummary clusterSummary = client.getClusterInfo();
TopologySummary topologySummary = null;
- for (TopologySummary ts: clusterSummary.get_topologies()) {
+ for (TopologySummary ts : clusterSummary.get_topologies()) {
if (topology.equals(ts.get_name())) {
topologySummary = ts;
break;
@@ -113,7 +50,7 @@ public class Monitor {
GetInfoOptions getInfoOpts = new GetInfoOptions();
getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts);
- for (ExecutorSummary es: info.get_executors()) {
+ for (ExecutorSummary es : info.get_executors()) {
components.add(es.get_component_id());
}
}
@@ -145,7 +82,7 @@ public class Monitor {
throw new IllegalArgumentException("stream name must be something");
}
- if ( !WATCH_TRANSFERRED.equals(_watch) && !WATCH_EMITTED.equals(_watch)) {
+ if (!WATCH_TRANSFERRED.equals(_watch) && !WATCH_EMITTED.equals(_watch)) {
throw new IllegalArgumentException("watch item must either be transferred or emitted");
}
System.out.println("topology\tcomponent\tparallelism\tstream\ttime-diff ms\t" + _watch + "\tthroughput (Kt/s)");
@@ -173,7 +110,7 @@ public class Monitor {
boolean streamFound = false;
ClusterSummary clusterSummary = client.getClusterInfo();
TopologySummary topologySummary = null;
- for (TopologySummary ts: clusterSummary.get_topologies()) {
+ for (TopologySummary ts : clusterSummary.get_topologies()) {
if (_topology.equals(ts.get_name())) {
topologySummary = ts;
break;
@@ -186,18 +123,18 @@ public class Monitor {
GetInfoOptions getInfoOpts = new GetInfoOptions();
getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts);
- for (ExecutorSummary es: info.get_executors()) {
+ for (ExecutorSummary es : info.get_executors()) {
if (_component.equals(es.get_component_id())) {
- componentParallelism ++;
+ componentParallelism++;
ExecutorStats stats = es.get_stats();
if (stats != null) {
- Map<String,Map<String,Long>> statted =
- WATCH_EMITTED.equals(_watch) ? stats.get_emitted() : stats.get_transferred();
- if ( statted != null) {
+ Map<String, Map<String, Long>> statted =
+ WATCH_EMITTED.equals(_watch) ? stats.get_emitted() : stats.get_transferred();
+ if (statted != null) {
Map<String, Long> e2 = statted.get(":all-time");
if (e2 != null) {
Long stream = e2.get(_stream);
- if (stream != null){
+ if (stream != null) {
streamFound = true;
totalStatted += stream;
}
@@ -226,8 +163,10 @@ public class Monitor {
long stattedDelta = totalStatted - state.getLastStatted();
state.setLastTime(now);
state.setLastStatted(totalStatted);
- double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double)stattedDelta/(double)timeDelta);
- System.out.println(_topology+"\t"+_component+"\t"+componentParallelism+"\t"+_stream+"\t"+timeDelta+"\t"+stattedDelta+"\t"+throughput);
+ double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double) stattedDelta / (double) timeDelta);
+ System.out.println(
+ _topology + "\t" + _component + "\t" + componentParallelism + "\t" + _stream + "\t" + timeDelta + "\t" + stattedDelta + "\t" +
+ throughput);
}
public void set_interval(int _interval) {
@@ -249,4 +188,68 @@ public class Monitor {
public void set_watch(String _watch) {
this._watch = _watch;
}
+
+ private static class MetricsState {
+ private long lastTime = 0;
+ private long lastStatted = 0;
+
+ private MetricsState(long lastTime, long lastStatted) {
+ this.lastTime = lastTime;
+ this.lastStatted = lastStatted;
+ }
+
+ public long getLastStatted() {
+ return lastStatted;
+ }
+
+ public void setLastStatted(long lastStatted) {
+ this.lastStatted = lastStatted;
+ }
+
+ public long getLastTime() {
+ return lastTime;
+ }
+
+ public void setLastTime(long lastTime) {
+ this.lastTime = lastTime;
+ }
+ }
+
+ private static class Poller {
+ private long startTime = 0;
+ private long pollMs = 0;
+
+ private Poller(long startTime, long pollMs) {
+ this.startTime = startTime;
+ this.pollMs = pollMs;
+ }
+
+ public long nextPoll() throws InterruptedException {
+ long now = System.currentTimeMillis();
+ long cycle = (now - startTime) / pollMs;
+ long wakeupTime = startTime + (pollMs * (cycle + 1));
+ long sleepTime = wakeupTime - now;
+ if (sleepTime > 0) {
+ Thread.sleep(sleepTime);
+ }
+ now = System.currentTimeMillis();
+ return now;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ public long getPollMs() {
+ return pollMs;
+ }
+
+ public void setPollMs(long pollMs) {
+ this.pollMs = pollMs;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java b/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java
index 0577261..db4c1d7 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/TopologySpoutLag.java
@@ -1,23 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.utils;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.generated.ComponentCommon;
import org.apache.storm.generated.ComponentObject;
@@ -28,27 +28,20 @@ import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
public class TopologySpoutLag {
// FIXME: This class can be moved to webapp once UI porting is done.
private static final String SPOUT_ID = "spoutId";
- private static final String SPOUT_TYPE= "spoutType";
+ private static final String SPOUT_TYPE = "spoutType";
private static final String SPOUT_LAG_RESULT = "spoutLagResult";
private static final String ERROR_INFO = "errorInfo";
private final static Logger logger = LoggerFactory.getLogger(TopologySpoutLag.class);
- public static Map<String, Map<String, Object>> lag (StormTopology stormTopology, Map<String, Object> topologyConf) {
+ public static Map<String, Map<String, Object>> lag(StormTopology stormTopology, Map<String, Object> topologyConf) {
Map<String, Map<String, Object>> result = new HashMap<>();
Map<String, SpoutSpec> spouts = stormTopology.get_spouts();
String className = null;
- for (Map.Entry<String, SpoutSpec> spout: spouts.entrySet()) {
+ for (Map.Entry<String, SpoutSpec> spout : spouts.entrySet()) {
try {
SpoutSpec spoutSpec = spout.getValue();
ComponentObject componentObject = spoutSpec.get_spout_object();
@@ -82,18 +75,18 @@ public class TopologySpoutLag {
}
}
- private static List<String> getCommandLineOptionsForNewKafkaSpout (Map<String, Object> jsonConf) {
+ private static List<String> getCommandLineOptionsForNewKafkaSpout(Map<String, Object> jsonConf) {
logger.debug("json configuration: {}", jsonConf);
List<String> commands = new ArrayList<>();
String configKeyPrefix = "config.";
commands.add("-t");
- commands.add((String)jsonConf.get(configKeyPrefix + "topics"));
+ commands.add((String) jsonConf.get(configKeyPrefix + "topics"));
commands.add("-g");
- commands.add((String)jsonConf.get(configKeyPrefix + "groupid"));
+ commands.add((String) jsonConf.get(configKeyPrefix + "groupid"));
commands.add("-b");
- commands.add((String)jsonConf.get(configKeyPrefix + "bootstrap.servers"));
- String securityProtocol = (String)jsonConf.get(configKeyPrefix + "security.protocol");
+ commands.add((String) jsonConf.get(configKeyPrefix + "bootstrap.servers"));
+ String securityProtocol = (String) jsonConf.get(configKeyPrefix + "security.protocol");
if (securityProtocol != null && !securityProtocol.isEmpty()) {
commands.add("-s");
commands.add(securityProtocol);
@@ -101,21 +94,21 @@ public class TopologySpoutLag {
return commands;
}
- private static List<String> getCommandLineOptionsForOldKafkaSpout (Map<String, Object> jsonConf, Map<String, Object> topologyConf) {
+ private static List<String> getCommandLineOptionsForOldKafkaSpout(Map<String, Object> jsonConf, Map<String, Object> topologyConf) {
logger.debug("json configuration: {}", jsonConf);
List<String> commands = new ArrayList<>();
String configKeyPrefix = "config.";
commands.add("-o");
commands.add("-t");
- commands.add((String)jsonConf.get(configKeyPrefix + "topics"));
+ commands.add((String) jsonConf.get(configKeyPrefix + "topics"));
commands.add("-n");
- commands.add((String)jsonConf.get(configKeyPrefix + "zkRoot"));
- String zkServers = (String)jsonConf.get(configKeyPrefix + "zkServers");
+ commands.add((String) jsonConf.get(configKeyPrefix + "zkRoot"));
+ String zkServers = (String) jsonConf.get(configKeyPrefix + "zkServers");
if (zkServers == null || zkServers.isEmpty()) {
StringBuilder zkServersBuilder = new StringBuilder();
Integer zkPort = ((Number) topologyConf.get(Config.STORM_ZOOKEEPER_PORT)).intValue();
- for (String zkServer: (List<String>) topologyConf.get(Config.STORM_ZOOKEEPER_SERVERS)) {
+ for (String zkServer : (List<String>) topologyConf.get(Config.STORM_ZOOKEEPER_SERVERS)) {
zkServersBuilder.append(zkServer + ":" + zkPort + ",");
}
zkServers = zkServersBuilder.toString();
@@ -124,12 +117,12 @@ public class TopologySpoutLag {
commands.add(zkServers);
if (jsonConf.get(configKeyPrefix + "leaders") != null) {
commands.add("-p");
- commands.add((String)jsonConf.get(configKeyPrefix + "partitions"));
+ commands.add((String) jsonConf.get(configKeyPrefix + "partitions"));
commands.add("-l");
- commands.add((String)jsonConf.get(configKeyPrefix + "leaders"));
+ commands.add((String) jsonConf.get(configKeyPrefix + "leaders"));
} else {
commands.add("-r");
- commands.add((String)jsonConf.get(configKeyPrefix + "zkNodeBrokers"));
+ commands.add((String) jsonConf.get(configKeyPrefix + "zkNodeBrokers"));
Boolean isWildCard = (Boolean) topologyConf.get("kafka.topic.wildcard.match");
if (isWildCard != null && isWildCard.booleanValue()) {
commands.add("-w");
@@ -138,7 +131,8 @@ public class TopologySpoutLag {
return commands;
}
- private static Map<String, Object> getLagResultForKafka (String spoutId, SpoutSpec spoutSpec, Map<String, Object> topologyConf, boolean old) throws IOException {
+ private static Map<String, Object> getLagResultForKafka(String spoutId, SpoutSpec spoutSpec, Map<String, Object> topologyConf,
+ boolean old) throws IOException {
ComponentCommon componentCommon = spoutSpec.get_common();
String json = componentCommon.get_json_conf();
Map<String, Object> result = null;
@@ -156,7 +150,8 @@ public class TopologySpoutLag {
} catch (ParseException e) {
throw new IOException(e);
}
- commands.addAll(old ? getCommandLineOptionsForOldKafkaSpout(jsonMap, topologyConf) : getCommandLineOptionsForNewKafkaSpout(jsonMap));
+ commands.addAll(
+ old ? getCommandLineOptionsForOldKafkaSpout(jsonMap, topologyConf) : getCommandLineOptionsForNewKafkaSpout(jsonMap));
logger.debug("Command to run: {}", commands);
@@ -187,11 +182,13 @@ public class TopologySpoutLag {
return kafkaSpoutLagInfo;
}
- private static Map<String, Object> getLagResultForNewKafkaSpout (String spoutId, SpoutSpec spoutSpec, Map<String, Object> topologyConf) throws IOException {
+ private static Map<String, Object> getLagResultForNewKafkaSpout(String spoutId, SpoutSpec spoutSpec,
+ Map<String, Object> topologyConf) throws IOException {
return getLagResultForKafka(spoutId, spoutSpec, topologyConf, false);
}
- private static Map<String, Object> getLagResultForOldKafkaSpout (String spoutId, SpoutSpec spoutSpec, Map<String, Object> topologyConf) throws IOException {
+ private static Map<String, Object> getLagResultForOldKafkaSpout(String spoutId, SpoutSpec spoutSpec,
+ Map<String, Object> topologyConf) throws IOException {
return getLagResultForKafka(spoutId, spoutSpec, topologyConf, true);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/MockAutoCred.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/MockAutoCred.java b/storm-core/test/jvm/org/apache/storm/MockAutoCred.java
index 1a746ca..5eede63 100644
--- a/storm-core/test/jvm/org/apache/storm/MockAutoCred.java
+++ b/storm-core/test/jvm/org/apache/storm/MockAutoCred.java
@@ -1,30 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm;
+import java.util.Map;
+import javax.security.auth.Subject;
import org.apache.storm.security.INimbusCredentialPlugin;
import org.apache.storm.security.auth.IAutoCredentials;
import org.apache.storm.security.auth.ICredentialsRenewer;
-import java.util.Map;
-
-import javax.security.auth.Subject;
-
/**
* mock implementation of INimbusCredentialPlugin,IAutoCredentials and ICredentialsRenewer for testing only.
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/command/RebalanceTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/command/RebalanceTest.java b/storm-core/test/jvm/org/apache/storm/command/RebalanceTest.java
index cec4958..e85beb5 100644
--- a/storm-core/test/jvm/org/apache/storm/command/RebalanceTest.java
+++ b/storm-core/test/jvm/org/apache/storm/command/RebalanceTest.java
@@ -1,29 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.command;
+import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
-import java.util.Map;
-
public class RebalanceTest {
@Test
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/command/SetLogLevelTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/command/SetLogLevelTest.java b/storm-core/test/jvm/org/apache/storm/command/SetLogLevelTest.java
index 4582371..5951752 100644
--- a/storm-core/test/jvm/org/apache/storm/command/SetLogLevelTest.java
+++ b/storm-core/test/jvm/org/apache/storm/command/SetLogLevelTest.java
@@ -1,30 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.command;
+import java.util.Map;
import org.apache.storm.generated.LogLevel;
import org.apache.storm.generated.LogLevelAction;
import org.junit.Assert;
import org.junit.Test;
-import java.util.Map;
-
public class SetLogLevelTest {
@Test
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/command/TestCLI.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/command/TestCLI.java b/storm-core/test/jvm/org/apache/storm/command/TestCLI.java
index d1caa3c..fd06cbe 100644
--- a/storm-core/test/jvm/org/apache/storm/command/TestCLI.java
+++ b/storm-core/test/jvm/org/apache/storm/command/TestCLI.java
@@ -1,28 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.command;
-import org.junit.Test;
-
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -31,22 +24,23 @@ public class TestCLI {
@Test
public void testSimple() throws Exception {
Map<String, Object> values = CLI.opt("a", "aa", null)
- .opt("b", "bb", 1, CLI.AS_INT)
- .opt("c", "cc", 1, CLI.AS_INT, CLI.FIRST_WINS)
- .opt("d", "dd", null, CLI.AS_STRING, CLI.INTO_LIST)
- .opt("e", "ee", null, CLI.AS_INT)
- .opt("f", "ff", null, new PairParse(), CLI.INTO_MAP)
- .arg("A")
- .arg("B", CLI.AS_INT)
- .parse("-a100", "--aa", "200", "-c2", "-b", "50", "--cc", "100", "A-VALUE", "1", "2", "3", "-b40", "-d1", "-d2", "-d3"
- , "-f", "key1=value1", "-f", "key2=value2");
+ .opt("b", "bb", 1, CLI.AS_INT)
+ .opt("c", "cc", 1, CLI.AS_INT, CLI.FIRST_WINS)
+ .opt("d", "dd", null, CLI.AS_STRING, CLI.INTO_LIST)
+ .opt("e", "ee", null, CLI.AS_INT)
+ .opt("f", "ff", null, new PairParse(), CLI.INTO_MAP)
+ .arg("A")
+ .arg("B", CLI.AS_INT)
+ .parse("-a100", "--aa", "200", "-c2", "-b", "50", "--cc", "100", "A-VALUE", "1", "2", "3", "-b40",
+ "-d1", "-d2", "-d3"
+ , "-f", "key1=value1", "-f", "key2=value2");
assertEquals(8, values.size());
- assertEquals("200", (String)values.get("a"));
- assertEquals((Integer)40, (Integer)values.get("b"));
- assertEquals((Integer)2, (Integer)values.get("c"));
+ assertEquals("200", (String) values.get("a"));
+ assertEquals((Integer) 40, (Integer) values.get("b"));
+ assertEquals((Integer) 2, (Integer) values.get("c"));
assertEquals(null, values.get("e"));
- List<String> d = (List<String>)values.get("d");
+ List<String> d = (List<String>) values.get("d");
assertEquals(3, d.size());
assertEquals("1", d.get(0));
assertEquals("2", d.get(1));
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/metric/FakeMetricConsumer.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/metric/FakeMetricConsumer.java b/storm-core/test/jvm/org/apache/storm/metric/FakeMetricConsumer.java
index 64ea8ac..23b1c8f 100644
--- a/storm-core/test/jvm/org/apache/storm/metric/FakeMetricConsumer.java
+++ b/storm-core/test/jvm/org/apache/storm/metric/FakeMetricConsumer.java
@@ -1,39 +1,39 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.metric;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.Multimap;
import com.google.common.collect.Table;
-
-import org.apache.storm.metric.api.IMetricsConsumer;
-import org.apache.storm.task.IErrorReporter;
-import org.apache.storm.task.TopologyContext;
-
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import org.apache.storm.metric.api.IMetricsConsumer;
+import org.apache.storm.task.IErrorReporter;
+import org.apache.storm.task.TopologyContext;
public class FakeMetricConsumer implements IMetricsConsumer {
public static final Table<String, String, Multimap<Integer, Object>> buffer = HashBasedTable.create();
+ public static Map<Integer, Collection<Object>> getTaskIdToBuckets(String componentName, String metricName) {
+ synchronized (buffer) {
+ Multimap<Integer, Object> taskIdToBuckets = buffer.get(componentName, metricName);
+ return (null != taskIdToBuckets) ? taskIdToBuckets.asMap() : null;
+ }
+ }
+
@Override
public void prepare(Map<String, Object> topoConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) {
synchronized (buffer) {
@@ -78,11 +78,4 @@ public class FakeMetricConsumer implements IMetricsConsumer {
}
return expanded;
}
-
- public static Map<Integer, Collection<Object>> getTaskIdToBuckets(String componentName, String metricName) {
- synchronized (buffer) {
- Multimap<Integer, Object> taskIdToBuckets = buffer.get(componentName, metricName);
- return (null != taskIdToBuckets) ? taskIdToBuckets.asMap() : null;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java b/storm-core/test/jvm/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java
index c1f082c..a7102c4 100644
--- a/storm-core/test/jvm/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java
+++ b/storm-core/test/jvm/org/apache/storm/nimbus/InMemoryTopologyActionNotifier.java
@@ -1,28 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.nimbus;
-import java.util.Map;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
-import java.util.HashMap;
+import java.util.Map;
-public class InMemoryTopologyActionNotifier implements ITopologyActionNotifierPlugin {
+public class InMemoryTopologyActionNotifier implements ITopologyActionNotifierPlugin {
//static to ensure eventhough the class is created using reflection we can still get
//the topology to actions
@@ -36,7 +31,7 @@ public class InMemoryTopologyActionNotifier implements ITopologyActionNotifierP
@Override
public synchronized void notify(String topologyName, String action) {
- if(!topologyToActions.containsKey(topologyName)) {
+ if (!topologyToActions.containsKey(topologyName)) {
topologyToActions.put(topologyName, new LinkedList<String>());
}
topologyToActions.get(topologyName).addLast(action);
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/serialization/SerializationTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/serialization/SerializationTest.java b/storm-core/test/jvm/org/apache/storm/serialization/SerializationTest.java
index 5fbf472..f122ff6 100644
--- a/storm-core/test/jvm/org/apache/storm/serialization/SerializationTest.java
+++ b/storm-core/test/jvm/org/apache/storm/serialization/SerializationTest.java
@@ -15,20 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.serialization;
import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.testing.TestSerObject;
import org.apache.storm.utils.Utils;
import org.junit.Assert;
import org.junit.Test;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
// FIXME: it should be moved to storm-client when serialization-test.clj can be removed
public class SerializationTest {
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java b/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java
index 1ca793b..d943bfb 100644
--- a/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java
+++ b/storm-core/test/jvm/org/apache/storm/stats/TestStatsUtil.java
@@ -1,35 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.stats;
-import java.util.Arrays;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.HashMap;
-
-import org.apache.storm.scheduler.WorkerSlot;
import org.apache.storm.generated.WorkerResources;
import org.apache.storm.generated.WorkerSummary;
-
-import org.junit.Before;
-import org.junit.Test;
+import org.apache.storm.scheduler.WorkerSlot;
import org.junit.Assert;
+import org.junit.Test;
public class TestStatsUtil {
@@ -42,7 +34,7 @@ public class TestStatsUtil {
private Map<String, String> nodeHost = new HashMap<String, String>();
private Map<WorkerSlot, WorkerResources> worker2Resources = new HashMap<WorkerSlot, WorkerResources>();
- private List<Long> makeExecutorId(int firstTask, int lastTask){
+ private List<Long> makeExecutorId(int firstTask, int lastTask) {
return Arrays.asList(new Long(firstTask), new Long(lastTask));
}
@@ -51,7 +43,7 @@ public class TestStatsUtil {
hostPort.add("node1");
hostPort.add(new Long(1));
- exec2NodePort.put(makeExecutorId(1,1), hostPort);
+ exec2NodePort.put(makeExecutorId(1, 1), hostPort);
nodeHost.put("node1", "host1");
nodeHost.put("node2", "host2");
@@ -119,9 +111,9 @@ public class TestStatsUtil {
}
private List<WorkerSummary> checkWorkerStats(boolean includeSys, boolean userAuthorized, String filterSupervisor) {
- List<WorkerSummary> summaries =
- StatsUtil.aggWorkerStats("my-storm-id", "my-storm-name",
- task2Component, beats, exec2NodePort, nodeHost, worker2Resources,
+ List<WorkerSummary> summaries =
+ StatsUtil.aggWorkerStats("my-storm-id", "my-storm-name",
+ task2Component, beats, exec2NodePort, nodeHost, worker2Resources,
includeSys, userAuthorized, filterSupervisor);
for (WorkerSummary ws : summaries) {
String host = ws.get_host();
@@ -178,8 +170,8 @@ public class TestStatsUtil {
@Test
public void aggWorkerStats() {
makeTopoInfo();
- List<WorkerSummary> summaries = checkWorkerStats(true /*include sys*/,
- true /*user authorized*/,
+ List<WorkerSummary> summaries = checkWorkerStats(true /*include sys*/,
+ true /*user authorized*/,
null /*filter supervisor*/);
WorkerSummary ws = getWorkerSummaryForPort(summaries, 1);
Assert.assertEquals(1, ws.get_component_to_num_tasks().size());
@@ -190,8 +182,8 @@ public class TestStatsUtil {
@Test
public void aggWorkerStatsWithSystemComponents() {
makeTopoInfoWithSysWorker();
- List<WorkerSummary> summaries = checkWorkerStats(true /*include sys*/,
- true /*user authorized*/,
+ List<WorkerSummary> summaries = checkWorkerStats(true /*include sys*/,
+ true /*user authorized*/,
null /*filter supervisor*/);
WorkerSummary ws = getWorkerSummaryForPort(summaries, 2);
// since we made sys components visible, the component map has all system components
@@ -205,8 +197,8 @@ public class TestStatsUtil {
@Test
public void aggWorkerStatsWithHiddenSystemComponents() {
makeTopoInfoWithSysWorker();
- List<WorkerSummary> summaries = checkWorkerStats(false /*DON'T include sys*/,
- true /*user authorized*/,
+ List<WorkerSummary> summaries = checkWorkerStats(false /*DON'T include sys*/,
+ true /*user authorized*/,
null /*filter supervisor*/);
WorkerSummary ws1 = getWorkerSummaryForPort(summaries, 1);
WorkerSummary ws2 = getWorkerSummaryForPort(summaries, 2);
@@ -219,8 +211,8 @@ public class TestStatsUtil {
@Test
public void aggWorkerStatsForUnauthorizedUser() {
makeTopoInfoWithSysWorker();
- List<WorkerSummary> summaries = checkWorkerStats(true /*include sys (should not matter)*/,
- false /*user NOT authorized*/,
+ List<WorkerSummary> summaries = checkWorkerStats(true /*include sys (should not matter)*/,
+ false /*user NOT authorized*/,
null /*filter supervisor*/);
WorkerSummary ws1 = getWorkerSummaryForPort(summaries, 1);
WorkerSummary ws2 = getWorkerSummaryForPort(summaries, 2);
@@ -233,8 +225,8 @@ public class TestStatsUtil {
@Test
public void aggWorkerStatsFilterSupervisor() {
makeTopoInfoWithMissingBeats();
- List<WorkerSummary> summaries = checkWorkerStats(true /*include sys*/,
- true /*user authorized*/,
+ List<WorkerSummary> summaries = checkWorkerStats(true /*include sys*/,
+ true /*user authorized*/,
"node3" /*filter supervisor*/);
WorkerSummary ws = getWorkerSummaryForPort(summaries, 3);
// only host3 should be returned given filter
@@ -247,8 +239,8 @@ public class TestStatsUtil {
@Test
public void aggWorkerStatsFilterSupervisorAndHideSystemComponents() {
makeTopoInfoWithMissingBeats();
- List<WorkerSummary> summaries = checkWorkerStats(false /*DON'T include sys*/,
- true /*user authorized*/,
+ List<WorkerSummary> summaries = checkWorkerStats(false /*DON'T include sys*/,
+ true /*user authorized*/,
"node3" /*filter supervisor*/);
WorkerSummary ws = getWorkerSummaryForPort(summaries, 3);
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ConfigUtilsInstaller.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ConfigUtilsInstaller.java b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ConfigUtilsInstaller.java
index bd39637..f76a01a 100644
--- a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ConfigUtilsInstaller.java
+++ b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ConfigUtilsInstaller.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.utils.staticmocking;
import org.apache.storm.utils.ConfigUtils;
@@ -35,7 +36,7 @@ public class ConfigUtilsInstaller implements AutoCloseable {
public void close() throws Exception {
if (ConfigUtils.setInstance(_oldInstance) != _curInstance) {
throw new IllegalStateException(
- "Instances of this resource must be closed in reverse order of opening.");
+ "Instances of this resource must be closed in reverse order of opening.");
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ReflectionUtilsInstaller.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ReflectionUtilsInstaller.java b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ReflectionUtilsInstaller.java
index 21f80bd..c2f1089 100644
--- a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ReflectionUtilsInstaller.java
+++ b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ReflectionUtilsInstaller.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.utils.staticmocking;
import org.apache.storm.utils.ReflectionUtils;
@@ -33,7 +34,7 @@ public class ReflectionUtilsInstaller implements AutoCloseable {
public void close() throws Exception {
if (ReflectionUtils.setInstance(_oldInstance) != _curInstance) {
throw new IllegalStateException(
- "Instances of this resource must be closed in reverse order of opening.");
+ "Instances of this resource must be closed in reverse order of opening.");
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ServerConfigUtilsInstaller.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ServerConfigUtilsInstaller.java b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ServerConfigUtilsInstaller.java
index eb81bc4..be05d0d 100644
--- a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ServerConfigUtilsInstaller.java
+++ b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/ServerConfigUtilsInstaller.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.utils.staticmocking;
import org.apache.storm.utils.ServerConfigUtils;
@@ -35,7 +36,7 @@ public class ServerConfigUtilsInstaller implements AutoCloseable {
public void close() throws Exception {
if (ServerConfigUtils.setInstance(_oldInstance) != _curInstance) {
throw new IllegalStateException(
- "Instances of this resource must be closed in reverse order of opening.");
+ "Instances of this resource must be closed in reverse order of opening.");
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/UtilsInstaller.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/UtilsInstaller.java b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/UtilsInstaller.java
index f700bd9..76b1624 100644
--- a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/UtilsInstaller.java
+++ b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/UtilsInstaller.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.utils.staticmocking;
import org.apache.storm.utils.Utils;
@@ -33,7 +34,7 @@ public class UtilsInstaller implements AutoCloseable {
public void close() throws Exception {
if (Utils.setInstance(_oldInstance) != _curInstance) {
throw new IllegalStateException(
- "Instances of this resource must be closed in reverse order of opening.");
+ "Instances of this resource must be closed in reverse order of opening.");
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/fc1cf09b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/package-info.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/package-info.java b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/package-info.java
index b41a2cd..00aa8a3 100644
--- a/storm-core/test/jvm/org/apache/storm/utils/staticmocking/package-info.java
+++ b/storm-core/test/jvm/org/apache/storm/utils/staticmocking/package-info.java
@@ -1,18 +1,61 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version
- * 2.0 (the "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- * implied. See the License for the specific language governing
- * permissions and limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * Provides implementations for testing static methods.
+ *
+ * This package should not exist and is only necessary while we need to mock static methods.
+ *
+ * To mock static methods in java, we use a singleton. The class to mock must implement <code>setInstance</code> static method that accepts
+ * an instance of the selfsame class and returns the previous instance that was set.
+ *
+ * Example:
+ *
+ * <code>
+ * public class MyClass { public static MyClass setInstance(MyClass c) { MyClass oldInstance = _instance; _instance = c; return oldInstance;
+ * }
+ *
+ * // Any method that we wish to mock must delegate to the singleton // instance's corresponding member method implementation public static
+ * int mockableFunction(String arg) { return _instance.mockableFunctionImpl(); }
+ *
+ * protected int mockableFunctionImpl(String arg) { return arg.size(); } }
+ * </code>
+ *
+ * Each class that could be mocked should have an Installer class defined in this package that sets the instance on construction and
+ * implements the
+ * <code>close</code> method of {@link java.lang.AutoCloseable}.
+ *
+ * Example:
+ *
+ * <code>
+ * class MyClassInstaller implementes AutoCloseable { private MyClass _oldInstance; private MyClass _curInstance;
+ *
+ * MyClassInstaller(MyClass instance) { _oldInstance = MyClass.setInstance(instance); _curInstance = instance; }
+ *
+ * @Override public void close() throws Exception { if (MyClass.setInstance(_oldInstance) != _curInstance) { throw new
+ * IllegalStateException( "Instances of this resource must be closed in reverse order of opening."); } } }
+ * </code>
+ *
+ * To write a test with the mocked class instantiate a child class that implements the close method, and use try-with-resources. For
+ * example:
+ *
+ * <code>
+ * MyClass mock = new MyClass() { protected int mockableFunctionImpl(String arg) { return 42; } };
+ *
+ * try(mock) { AssertEqual(42, MyClass.mockableFunction("not 42 characters")); };
+ * </code>
+ *
+ *
+ * The resulting code remains thread-unsafe.
+ *
+ * This class should be removed when troublesome static methods have been replaced in the code.
*/
/**
@@ -92,4 +135,5 @@
* This class should be removed when troublesome static methods have been
* replaced in the code.
*/
+
package org.apache.storm.utils.staticmocking;
[37/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-starter
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java
index 245c552..8ee4f22 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/RankingsTest.java
@@ -1,368 +1,406 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.tools;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import java.util.List;
import org.jmock.lib.concurrent.Blitzer;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import java.util.List;
-
import static org.fest.assertions.api.Assertions.assertThat;
public class RankingsTest {
- private static final int ANY_TOPN = 42;
- private static final Rankable ANY_RANKABLE = new RankableObjectWithFields("someObject", ANY_TOPN);
- private static final Rankable ZERO = new RankableObjectWithFields("ZERO_COUNT", 0);
- private static final Rankable A = new RankableObjectWithFields("A", 1);
- private static final Rankable B = new RankableObjectWithFields("B", 2);
- private static final Rankable C = new RankableObjectWithFields("C", 3);
- private static final Rankable D = new RankableObjectWithFields("D", 4);
- private static final Rankable E = new RankableObjectWithFields("E", 5);
- private static final Rankable F = new RankableObjectWithFields("F", 6);
- private static final Rankable G = new RankableObjectWithFields("G", 7);
- private static final Rankable H = new RankableObjectWithFields("H", 8);
-
- @DataProvider
- public Object[][] illegalTopNData() {
- return new Object[][]{ { 0 }, { -1 }, { -2 }, { -10 } };
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopNData")
- public void constructorWithNegativeOrZeroTopNShouldThrowIAE(int topN) {
- new Rankings(topN);
- }
-
- @DataProvider
- public Object[][] copyRankingsData() {
- return new Object[][]{ { 5, Lists.newArrayList(A, B, C) }, { 2, Lists.newArrayList(A, B, C, D) },
- { 1, Lists.newArrayList() }, { 1, Lists.newArrayList(A) }, { 1, Lists.newArrayList(A, B) } };
- }
-
- @Test(dataProvider = "copyRankingsData")
- public void copyConstructorShouldReturnCopy(int topN, List<Rankable> rankables) {
- // given
- Rankings rankings = new Rankings(topN);
- for (Rankable r : rankables) {
- rankings.updateWith(r);
+ private static final int ANY_TOPN = 42;
+ private static final Rankable ANY_RANKABLE = new RankableObjectWithFields("someObject", ANY_TOPN);
+ private static final Rankable ZERO = new RankableObjectWithFields("ZERO_COUNT", 0);
+ private static final Rankable A = new RankableObjectWithFields("A", 1);
+ private static final Rankable B = new RankableObjectWithFields("B", 2);
+ private static final Rankable C = new RankableObjectWithFields("C", 3);
+ private static final Rankable D = new RankableObjectWithFields("D", 4);
+ private static final Rankable E = new RankableObjectWithFields("E", 5);
+ private static final Rankable F = new RankableObjectWithFields("F", 6);
+ private static final Rankable G = new RankableObjectWithFields("G", 7);
+ private static final Rankable H = new RankableObjectWithFields("H", 8);
+
+ @DataProvider
+ public Object[][] illegalTopNData() {
+ return new Object[][]{ { 0 }, { -1 }, { -2 }, { -10 } };
}
- // when
- Rankings copy = new Rankings(rankings);
-
- // then
- assertThat(copy.maxSize()).isEqualTo(rankings.maxSize());
- assertThat(copy.getRankings()).isEqualTo(rankings.getRankings());
- }
-
- @DataProvider
- public Object[][] defensiveCopyRankingsData() {
- return new Object[][]{ { 5, Lists.newArrayList(A, B, C), Lists.newArrayList(D) }, { 2, Lists.newArrayList(A, B, C,
- D), Lists.newArrayList(E, F) }, { 1, Lists.newArrayList(), Lists.newArrayList(A) }, { 1, Lists.newArrayList(A),
- Lists.newArrayList(B) }, { 1, Lists.newArrayList(ZERO), Lists.newArrayList(B) }, { 1, Lists.newArrayList(ZERO),
- Lists.newArrayList() } };
- }
-
- @Test(dataProvider = "defensiveCopyRankingsData")
- public void copyConstructorShouldReturnDefensiveCopy(int topN, List<Rankable> rankables, List<Rankable> changes) {
- // given
- Rankings original = new Rankings(topN);
- for (Rankable r : rankables) {
- original.updateWith(r);
+ @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalTopNData")
+ public void constructorWithNegativeOrZeroTopNShouldThrowIAE(int topN) {
+ new Rankings(topN);
}
- int expSize = original.size();
- List<Rankable> expRankings = original.getRankings();
- // when
- Rankings copy = new Rankings(original);
- for (Rankable r : changes) {
- copy.updateWith(r);
+ @DataProvider
+ public Object[][] copyRankingsData() {
+ return new Object[][]{
+ { 5, Lists.newArrayList(A, B, C) }, { 2, Lists.newArrayList(A, B, C, D) },
+ { 1, Lists.newArrayList() }, { 1, Lists.newArrayList(A) }, { 1, Lists.newArrayList(A, B) }
+ };
}
- // then
- assertThat(original.size()).isEqualTo(expSize);
- assertThat(original.getRankings()).isEqualTo(expRankings);
- }
-
- @DataProvider
- public Object[][] legalTopNData() {
- return new Object[][]{ { 1 }, { 2 }, { 1000 }, { 1000000 } };
- }
-
- @Test(dataProvider = "legalTopNData")
- public void constructorWithPositiveTopNShouldBeOk(int topN) {
- // given/when
- Rankings rankings = new Rankings(topN);
-
- // then
- assertThat(rankings.maxSize()).isEqualTo(topN);
- }
-
- @Test
- public void shouldHaveDefaultConstructor() {
- new Rankings();
- }
-
- @Test
- public void defaultConstructorShouldSetPositiveTopN() {
- // given/when
- Rankings rankings = new Rankings();
-
- // then
- assertThat(rankings.maxSize()).isGreaterThan(0);
- }
-
- @DataProvider
- public Object[][] rankingsGrowData() {
- return new Object[][]{ { 2, Lists.newArrayList(new RankableObjectWithFields("A", 1), new RankableObjectWithFields(
- "B", 2), new RankableObjectWithFields("C", 3)) }, { 2, Lists.newArrayList(new RankableObjectWithFields("A", 1),
- new RankableObjectWithFields("B", 2), new RankableObjectWithFields("C", 3), new RankableObjectWithFields("D",
- 4)) } };
- }
-
- @Test(dataProvider = "rankingsGrowData")
- public void sizeOfRankingsShouldNotGrowBeyondTopN(int topN, List<Rankable> rankables) {
- // sanity check of the provided test data
- assertThat(rankables.size()).overridingErrorMessage(
- "The supplied test data is not correct: the number of rankables <%d> should be greater than <%d>",
- rankables.size(), topN).isGreaterThan(topN);
-
- // given
- Rankings rankings = new Rankings(topN);
-
- // when
- for (Rankable r : rankables) {
- rankings.updateWith(r);
+ @Test(dataProvider = "copyRankingsData")
+ public void copyConstructorShouldReturnCopy(int topN, List<Rankable> rankables) {
+ // given
+ Rankings rankings = new Rankings(topN);
+ for (Rankable r : rankables) {
+ rankings.updateWith(r);
+ }
+
+ // when
+ Rankings copy = new Rankings(rankings);
+
+ // then
+ assertThat(copy.maxSize()).isEqualTo(rankings.maxSize());
+ assertThat(copy.getRankings()).isEqualTo(rankings.getRankings());
}
- // then
- assertThat(rankings.size()).isLessThanOrEqualTo(rankings.maxSize());
- }
-
- @DataProvider
- public Object[][] simulatedRankingsData() {
- return new Object[][]{ { Lists.newArrayList(A), Lists.newArrayList(A) }, { Lists.newArrayList(B, D, A, C),
- Lists.newArrayList(D, C, B, A) }, { Lists.newArrayList(B, F, A, C, D, E), Lists.newArrayList(F, E, D, C, B,
- A) }, { Lists.newArrayList(G, B, F, A, C, D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A) } };
- }
-
- @Test(dataProvider = "simulatedRankingsData")
- public void shouldCorrectlyRankWhenUpdatedWithRankables(List<Rankable> unsorted, List<Rankable> expSorted) {
- // given
- Rankings rankings = new Rankings(unsorted.size());
-
- // when
- for (Rankable r : unsorted) {
- rankings.updateWith(r);
+ @DataProvider
+ public Object[][] defensiveCopyRankingsData() {
+ return new Object[][]{
+ { 5, Lists.newArrayList(A, B, C), Lists.newArrayList(D) }, {
+ 2, Lists.newArrayList(A, B, C,
+ D), Lists.newArrayList(E, F)
+ }, { 1, Lists.newArrayList(), Lists.newArrayList(A) }, {
+ 1, Lists.newArrayList(A),
+ Lists.newArrayList(B)
+ }, { 1, Lists.newArrayList(ZERO), Lists.newArrayList(B) }, {
+ 1, Lists.newArrayList(ZERO),
+ Lists.newArrayList()
+ }
+ };
}
- // then
- assertThat(rankings.getRankings()).isEqualTo(expSorted);
- }
-
- @Test(dataProvider = "simulatedRankingsData")
- public void shouldCorrectlyRankWhenEmptyAndUpdatedWithOtherRankings(List<Rankable> unsorted,
- List<Rankable> expSorted) {
- // given
- Rankings rankings = new Rankings(unsorted.size());
- Rankings otherRankings = new Rankings(rankings.maxSize());
- for (Rankable r : unsorted) {
- otherRankings.updateWith(r);
+ @Test(dataProvider = "defensiveCopyRankingsData")
+ public void copyConstructorShouldReturnDefensiveCopy(int topN, List<Rankable> rankables, List<Rankable> changes) {
+ // given
+ Rankings original = new Rankings(topN);
+ for (Rankable r : rankables) {
+ original.updateWith(r);
+ }
+ int expSize = original.size();
+ List<Rankable> expRankings = original.getRankings();
+
+ // when
+ Rankings copy = new Rankings(original);
+ for (Rankable r : changes) {
+ copy.updateWith(r);
+ }
+
+ // then
+ assertThat(original.size()).isEqualTo(expSize);
+ assertThat(original.getRankings()).isEqualTo(expRankings);
}
- // when
- rankings.updateWith(otherRankings);
+ @DataProvider
+ public Object[][] legalTopNData() {
+ return new Object[][]{ { 1 }, { 2 }, { 1000 }, { 1000000 } };
+ }
- // then
- assertThat(rankings.getRankings()).isEqualTo(expSorted);
- }
+ @Test(dataProvider = "legalTopNData")
+ public void constructorWithPositiveTopNShouldBeOk(int topN) {
+ // given/when
+ Rankings rankings = new Rankings(topN);
- @Test(dataProvider = "simulatedRankingsData")
- public void shouldCorrectlyRankWhenUpdatedWithEmptyOtherRankings(List<Rankable> unsorted, List<Rankable> expSorted) {
- // given
- Rankings rankings = new Rankings(unsorted.size());
- for (Rankable r : unsorted) {
- rankings.updateWith(r);
+ // then
+ assertThat(rankings.maxSize()).isEqualTo(topN);
}
- Rankings emptyRankings = new Rankings(ANY_TOPN);
-
- // when
- rankings.updateWith(emptyRankings);
-
- // then
- assertThat(rankings.getRankings()).isEqualTo(expSorted);
- }
-
- @DataProvider
- public Object[][] simulatedRankingsAndOtherRankingsData() {
- return new Object[][]{ { Lists.newArrayList(A), Lists.newArrayList(A), Lists.newArrayList(A) },
- { Lists.newArrayList(A, C), Lists.newArrayList(B, D), Lists.newArrayList(D, C, B, A) }, { Lists.newArrayList(B,
- F, A), Lists.newArrayList(C, D, E), Lists.newArrayList(F, E, D, C, B, A) }, { Lists.newArrayList(G, B, F, A, C),
- Lists.newArrayList(D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A) } };
- }
-
- @Test(dataProvider = "simulatedRankingsAndOtherRankingsData")
- public void shouldCorrectlyRankWhenNotEmptyAndUpdatedWithOtherRankings(List<Rankable> unsorted,
- List<Rankable> unsortedForOtherRankings, List<Rankable> expSorted) {
- // given
- Rankings rankings = new Rankings(expSorted.size());
- for (Rankable r : unsorted) {
- rankings.updateWith(r);
+
+ @Test
+ public void shouldHaveDefaultConstructor() {
+ new Rankings();
}
- Rankings otherRankings = new Rankings(unsortedForOtherRankings.size());
- for (Rankable r : unsortedForOtherRankings) {
- otherRankings.updateWith(r);
+
+ @Test
+ public void defaultConstructorShouldSetPositiveTopN() {
+ // given/when
+ Rankings rankings = new Rankings();
+
+ // then
+ assertThat(rankings.maxSize()).isGreaterThan(0);
}
- // when
- rankings.updateWith(otherRankings);
-
- // then
- assertThat(rankings.getRankings()).isEqualTo(expSorted);
- }
-
- @DataProvider
- public Object[][] duplicatesData() {
- Rankable A1 = new RankableObjectWithFields("A", 1);
- Rankable A2 = new RankableObjectWithFields("A", 2);
- Rankable A3 = new RankableObjectWithFields("A", 3);
- return new Object[][]{ { Lists.newArrayList(ANY_RANKABLE, ANY_RANKABLE, ANY_RANKABLE) }, { Lists.newArrayList(A1,
- A2, A3) }, };
- }
-
- @Test(dataProvider = "duplicatesData")
- public void shouldNotRankDuplicateObjectsMoreThanOnce(List<Rankable> duplicates) {
- // given
- Rankings rankings = new Rankings(duplicates.size());
-
- // when
- for (Rankable r : duplicates) {
- rankings.updateWith(r);
+ @DataProvider
+ public Object[][] rankingsGrowData() {
+ return new Object[][]{
+ {
+ 2, Lists.newArrayList(new RankableObjectWithFields("A", 1), new RankableObjectWithFields(
+ "B", 2), new RankableObjectWithFields("C", 3))
+ }, {
+ 2, Lists.newArrayList(new RankableObjectWithFields("A", 1),
+ new RankableObjectWithFields("B", 2), new RankableObjectWithFields("C", 3),
+ new RankableObjectWithFields("D",
+ 4))
+ }
+ };
}
- // then
- assertThat(rankings.size()).isEqualTo(1);
- }
-
- @DataProvider
- public Object[][] removeZeroRankingsData() {
- return new Object[][]{ { Lists.newArrayList(A, ZERO), Lists.newArrayList(A) }, { Lists.newArrayList(A),
- Lists.newArrayList(A) }, { Lists.newArrayList(ZERO, A), Lists.newArrayList(A) }, { Lists.newArrayList(ZERO),
- Lists.newArrayList() }, { Lists.newArrayList(ZERO, new RankableObjectWithFields("ZERO2", 0)),
- Lists.newArrayList() }, { Lists.newArrayList(B, ZERO, new RankableObjectWithFields("ZERO2", 0), D,
- new RankableObjectWithFields("ZERO3", 0), new RankableObjectWithFields("ZERO4", 0), C), Lists.newArrayList(D, C,
- B) }, { Lists.newArrayList(A, ZERO, B), Lists.newArrayList(B, A) } };
- }
-
- @Test(dataProvider = "removeZeroRankingsData")
- public void shouldRemoveZeroCounts(List<Rankable> unsorted, List<Rankable> expSorted) {
- // given
- Rankings rankings = new Rankings(unsorted.size());
- for (Rankable r : unsorted) {
- rankings.updateWith(r);
+ @Test(dataProvider = "rankingsGrowData")
+ public void sizeOfRankingsShouldNotGrowBeyondTopN(int topN, List<Rankable> rankables) {
+ // sanity check of the provided test data
+ assertThat(rankables.size()).overridingErrorMessage(
+ "The supplied test data is not correct: the number of rankables <%d> should be greater than <%d>",
+ rankables.size(), topN).isGreaterThan(topN);
+
+ // given
+ Rankings rankings = new Rankings(topN);
+
+ // when
+ for (Rankable r : rankables) {
+ rankings.updateWith(r);
+ }
+
+ // then
+ assertThat(rankings.size()).isLessThanOrEqualTo(rankings.maxSize());
}
- // when
- rankings.pruneZeroCounts();
-
- // then
- assertThat(rankings.getRankings()).isEqualTo(expSorted);
- }
-
- @Test
- public void updatingWithNewRankablesShouldBeThreadSafe() throws InterruptedException {
- // given
- final List<Rankable> entries = ImmutableList.of(A, B, C, D);
- final Rankings rankings = new Rankings(entries.size());
-
- // We are capturing exceptions thrown in Blitzer's child threads into this data structure so that we can properly
- // pass/fail this test. The reason is that Blitzer doesn't report exceptions, which is a known bug in Blitzer
- // (JMOCK-263). See https://github.com/jmock-developers/jmock-library/issues/22 for more information.
- final List<Exception> exceptions = Lists.newArrayList();
- Blitzer blitzer = new Blitzer(1000);
-
- // when
- blitzer.blitz(new Runnable() {
- public void run() {
- for (Rankable r : entries) {
- try {
+ @DataProvider
+ public Object[][] simulatedRankingsData() {
+ return new Object[][]{
+ { Lists.newArrayList(A), Lists.newArrayList(A) }, {
+ Lists.newArrayList(B, D, A, C),
+ Lists.newArrayList(D, C, B, A)
+ }, {
+ Lists.newArrayList(B, F, A, C, D, E), Lists.newArrayList(F, E, D, C, B,
+ A)
+ }, { Lists.newArrayList(G, B, F, A, C, D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A) }
+ };
+ }
+
+ @Test(dataProvider = "simulatedRankingsData")
+ public void shouldCorrectlyRankWhenUpdatedWithRankables(List<Rankable> unsorted, List<Rankable> expSorted) {
+ // given
+ Rankings rankings = new Rankings(unsorted.size());
+
+ // when
+ for (Rankable r : unsorted) {
+ rankings.updateWith(r);
+ }
+
+ // then
+ assertThat(rankings.getRankings()).isEqualTo(expSorted);
+ }
+
+ @Test(dataProvider = "simulatedRankingsData")
+ public void shouldCorrectlyRankWhenEmptyAndUpdatedWithOtherRankings(List<Rankable> unsorted,
+ List<Rankable> expSorted) {
+ // given
+ Rankings rankings = new Rankings(unsorted.size());
+ Rankings otherRankings = new Rankings(rankings.maxSize());
+ for (Rankable r : unsorted) {
+ otherRankings.updateWith(r);
+ }
+
+ // when
+ rankings.updateWith(otherRankings);
+
+ // then
+ assertThat(rankings.getRankings()).isEqualTo(expSorted);
+ }
+
+ @Test(dataProvider = "simulatedRankingsData")
+ public void shouldCorrectlyRankWhenUpdatedWithEmptyOtherRankings(List<Rankable> unsorted, List<Rankable> expSorted) {
+ // given
+ Rankings rankings = new Rankings(unsorted.size());
+ for (Rankable r : unsorted) {
rankings.updateWith(r);
- }
- catch (RuntimeException e) {
- synchronized(exceptions) {
- exceptions.add(e);
- }
- }
}
- }
- });
- blitzer.shutdown();
-
- // then
- //
- if (!exceptions.isEmpty()) {
- for (Exception e : exceptions) {
- System.err.println(Throwables.getStackTraceAsString(e));
- }
+ Rankings emptyRankings = new Rankings(ANY_TOPN);
+
+ // when
+ rankings.updateWith(emptyRankings);
+
+ // then
+ assertThat(rankings.getRankings()).isEqualTo(expSorted);
}
- assertThat(exceptions).isEmpty();
- }
-
- @Test(dataProvider = "copyRankingsData")
- public void copyShouldReturnCopy(int topN, List<Rankable> rankables) {
- // given
- Rankings rankings = new Rankings(topN);
- for (Rankable r : rankables) {
- rankings.updateWith(r);
+
+ @DataProvider
+ public Object[][] simulatedRankingsAndOtherRankingsData() {
+ return new Object[][]{
+ { Lists.newArrayList(A), Lists.newArrayList(A), Lists.newArrayList(A) },
+ { Lists.newArrayList(A, C), Lists.newArrayList(B, D), Lists.newArrayList(D, C, B, A) }, {
+ Lists.newArrayList(B,
+ F, A), Lists.newArrayList(C, D, E), Lists.newArrayList(F, E, D, C, B, A)
+ }, {
+ Lists.newArrayList(G, B, F, A, C),
+ Lists.newArrayList(D, E, H), Lists.newArrayList(H, G, F, E, D, C, B, A)
+ }
+ };
}
- // when
- Rankings copy = rankings.copy();
+ @Test(dataProvider = "simulatedRankingsAndOtherRankingsData")
+ public void shouldCorrectlyRankWhenNotEmptyAndUpdatedWithOtherRankings(List<Rankable> unsorted,
+ List<Rankable> unsortedForOtherRankings,
+ List<Rankable> expSorted) {
+ // given
+ Rankings rankings = new Rankings(expSorted.size());
+ for (Rankable r : unsorted) {
+ rankings.updateWith(r);
+ }
+ Rankings otherRankings = new Rankings(unsortedForOtherRankings.size());
+ for (Rankable r : unsortedForOtherRankings) {
+ otherRankings.updateWith(r);
+ }
- // then
- assertThat(copy.maxSize()).isEqualTo(rankings.maxSize());
- assertThat(copy.getRankings()).isEqualTo(rankings.getRankings());
- }
+ // when
+ rankings.updateWith(otherRankings);
- @Test(dataProvider = "defensiveCopyRankingsData")
- public void copyShouldReturnDefensiveCopy(int topN, List<Rankable> rankables, List<Rankable> changes) {
- // given
- Rankings original = new Rankings(topN);
- for (Rankable r : rankables) {
- original.updateWith(r);
+ // then
+ assertThat(rankings.getRankings()).isEqualTo(expSorted);
}
- int expSize = original.size();
- List<Rankable> expRankings = original.getRankings();
- // when
- Rankings copy = original.copy();
- for (Rankable r : changes) {
- copy.updateWith(r);
+ @DataProvider
+ public Object[][] duplicatesData() {
+ Rankable A1 = new RankableObjectWithFields("A", 1);
+ Rankable A2 = new RankableObjectWithFields("A", 2);
+ Rankable A3 = new RankableObjectWithFields("A", 3);
+ return new Object[][]{
+ { Lists.newArrayList(ANY_RANKABLE, ANY_RANKABLE, ANY_RANKABLE) }, {
+ Lists.newArrayList(A1,
+ A2, A3)
+ },
+ };
}
- copy.pruneZeroCounts();
- // then
- assertThat(original.size()).isEqualTo(expSize);
- assertThat(original.getRankings()).isEqualTo(expRankings);
- }
+ @Test(dataProvider = "duplicatesData")
+ public void shouldNotRankDuplicateObjectsMoreThanOnce(List<Rankable> duplicates) {
+ // given
+ Rankings rankings = new Rankings(duplicates.size());
+
+ // when
+ for (Rankable r : duplicates) {
+ rankings.updateWith(r);
+ }
+
+ // then
+ assertThat(rankings.size()).isEqualTo(1);
+ }
+
+ @DataProvider
+ public Object[][] removeZeroRankingsData() {
+ return new Object[][]{
+ { Lists.newArrayList(A, ZERO), Lists.newArrayList(A) }, {
+ Lists.newArrayList(A),
+ Lists.newArrayList(A)
+ }, { Lists.newArrayList(ZERO, A), Lists.newArrayList(A) }, {
+ Lists.newArrayList(ZERO),
+ Lists.newArrayList()
+ }, {
+ Lists.newArrayList(ZERO, new RankableObjectWithFields("ZERO2", 0)),
+ Lists.newArrayList()
+ }, {
+ Lists.newArrayList(B, ZERO, new RankableObjectWithFields("ZERO2", 0), D,
+ new RankableObjectWithFields("ZERO3", 0), new RankableObjectWithFields("ZERO4", 0), C), Lists.newArrayList(D,
+ C,
+ B)
+ }, { Lists.newArrayList(A, ZERO, B), Lists.newArrayList(B, A) }
+ };
+ }
+
+ @Test(dataProvider = "removeZeroRankingsData")
+ public void shouldRemoveZeroCounts(List<Rankable> unsorted, List<Rankable> expSorted) {
+ // given
+ Rankings rankings = new Rankings(unsorted.size());
+ for (Rankable r : unsorted) {
+ rankings.updateWith(r);
+ }
+
+ // when
+ rankings.pruneZeroCounts();
+
+ // then
+ assertThat(rankings.getRankings()).isEqualTo(expSorted);
+ }
+
+ @Test
+ public void updatingWithNewRankablesShouldBeThreadSafe() throws InterruptedException {
+ // given
+ final List<Rankable> entries = ImmutableList.of(A, B, C, D);
+ final Rankings rankings = new Rankings(entries.size());
+
+ // We are capturing exceptions thrown in Blitzer's child threads into this data structure so that we can properly
+ // pass/fail this test. The reason is that Blitzer doesn't report exceptions, which is a known bug in Blitzer
+ // (JMOCK-263). See https://github.com/jmock-developers/jmock-library/issues/22 for more information.
+ final List<Exception> exceptions = Lists.newArrayList();
+ Blitzer blitzer = new Blitzer(1000);
+
+ // when
+ blitzer.blitz(new Runnable() {
+ public void run() {
+ for (Rankable r : entries) {
+ try {
+ rankings.updateWith(r);
+ } catch (RuntimeException e) {
+ synchronized (exceptions) {
+ exceptions.add(e);
+ }
+ }
+ }
+ }
+ });
+ blitzer.shutdown();
+
+ // then
+ //
+ if (!exceptions.isEmpty()) {
+ for (Exception e : exceptions) {
+ System.err.println(Throwables.getStackTraceAsString(e));
+ }
+ }
+ assertThat(exceptions).isEmpty();
+ }
+
+ @Test(dataProvider = "copyRankingsData")
+ public void copyShouldReturnCopy(int topN, List<Rankable> rankables) {
+ // given
+ Rankings rankings = new Rankings(topN);
+ for (Rankable r : rankables) {
+ rankings.updateWith(r);
+ }
+
+ // when
+ Rankings copy = rankings.copy();
+
+ // then
+ assertThat(copy.maxSize()).isEqualTo(rankings.maxSize());
+ assertThat(copy.getRankings()).isEqualTo(rankings.getRankings());
+ }
+
+ @Test(dataProvider = "defensiveCopyRankingsData")
+ public void copyShouldReturnDefensiveCopy(int topN, List<Rankable> rankables, List<Rankable> changes) {
+ // given
+ Rankings original = new Rankings(topN);
+ for (Rankable r : rankables) {
+ original.updateWith(r);
+ }
+ int expSize = original.size();
+ List<Rankable> expRankings = original.getRankings();
+
+ // when
+ Rankings copy = original.copy();
+ for (Rankable r : changes) {
+ copy.updateWith(r);
+ }
+ copy.pruneZeroCounts();
+
+ // then
+ assertThat(original.size()).isEqualTo(expSize);
+ assertThat(original.getRankings()).isEqualTo(expRankings);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java
index e7b71cf..3bdb4ff 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlidingWindowCounterTest.java
@@ -1,106 +1,103 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.tools;
+import java.util.Map;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import java.util.Map;
-
import static org.fest.assertions.api.Assertions.assertThat;
public class SlidingWindowCounterTest {
- private static final int ANY_WINDOW_LENGTH_IN_SLOTS = 2;
- private static final Object ANY_OBJECT = "ANY_OBJECT";
-
- @DataProvider
- public Object[][] illegalWindowLengths() {
- return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 }, { 1 } };
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalWindowLengths")
- public void lessThanTwoSlotsShouldThrowIAE(int windowLengthInSlots) {
- new SlidingWindowCounter<Object>(windowLengthInSlots);
- }
-
- @DataProvider
- public Object[][] legalWindowLengths() {
- return new Object[][]{ { 2 }, { 3 }, { 20 } };
- }
-
- @Test(dataProvider = "legalWindowLengths")
- public void twoOrMoreSlotsShouldBeValid(int windowLengthInSlots) {
- new SlidingWindowCounter<Object>(windowLengthInSlots);
- }
-
- @Test
- public void newInstanceShouldHaveEmptyCounts() {
- // given
- SlidingWindowCounter<Object> counter = new SlidingWindowCounter<Object>(ANY_WINDOW_LENGTH_IN_SLOTS);
-
- // when
- Map<Object, Long> counts = counter.getCountsThenAdvanceWindow();
-
- // then
- assertThat(counts).isEmpty();
- }
-
- @DataProvider
- public Object[][] simulatedCounterIterations() {
- return new Object[][]{ { 2, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 2, 0, 1, 1, 0, 0 } },
- { 3, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 2, 1, 1, 1, 0 } },
- { 4, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 5, 3, 1, 1, 1 } },
- { 5, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 5, 6, 3, 1, 1 } },
- { 5, new int[]{ 3, 11, 5, 13, 7, 17, 0, 3, 50, 600, 7000 },
- new long[]{ 3, 14, 19, 32, 39, 53, 42, 40, 77, 670, 7653 } }, };
- }
-
- @Test(dataProvider = "simulatedCounterIterations")
- public void testCounterWithSimulatedRuns(int windowLengthInSlots, int[] incrementsPerIteration,
- long[] expCountsPerIteration) {
- // given
- SlidingWindowCounter<Object> counter = new SlidingWindowCounter<Object>(windowLengthInSlots);
- int numIterations = incrementsPerIteration.length;
-
- for (int i = 0; i < numIterations; i++) {
- int numIncrements = incrementsPerIteration[i];
- long expCounts = expCountsPerIteration[i];
- // Objects are absent if they were zero both this iteration
- // and the last -- if only this one, we need to report zero.
- boolean expAbsent = ((expCounts == 0) && ((i == 0) || (expCountsPerIteration[i - 1] == 0)));
-
- // given (for this iteration)
- for (int j = 0; j < numIncrements; j++) {
- counter.incrementCount(ANY_OBJECT);
- }
-
- // when (for this iteration)
- Map<Object, Long> counts = counter.getCountsThenAdvanceWindow();
-
- // then (for this iteration)
- if (expAbsent) {
- assertThat(counts).doesNotContainKey(ANY_OBJECT);
- }
- else {
- assertThat(counts.get(ANY_OBJECT)).isEqualTo(expCounts);
- }
+ private static final int ANY_WINDOW_LENGTH_IN_SLOTS = 2;
+ private static final Object ANY_OBJECT = "ANY_OBJECT";
+
+ @DataProvider
+ public Object[][] illegalWindowLengths() {
+ return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 }, { 1 } };
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalWindowLengths")
+ public void lessThanTwoSlotsShouldThrowIAE(int windowLengthInSlots) {
+ new SlidingWindowCounter<Object>(windowLengthInSlots);
+ }
+
+ @DataProvider
+ public Object[][] legalWindowLengths() {
+ return new Object[][]{ { 2 }, { 3 }, { 20 } };
+ }
+
+ @Test(dataProvider = "legalWindowLengths")
+ public void twoOrMoreSlotsShouldBeValid(int windowLengthInSlots) {
+ new SlidingWindowCounter<Object>(windowLengthInSlots);
+ }
+
+ @Test
+ public void newInstanceShouldHaveEmptyCounts() {
+ // given
+ SlidingWindowCounter<Object> counter = new SlidingWindowCounter<Object>(ANY_WINDOW_LENGTH_IN_SLOTS);
+
+ // when
+ Map<Object, Long> counts = counter.getCountsThenAdvanceWindow();
+
+ // then
+ assertThat(counts).isEmpty();
+ }
+
+ @DataProvider
+ public Object[][] simulatedCounterIterations() {
+ return new Object[][]{
+ { 2, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 2, 0, 1, 1, 0, 0 } },
+ { 3, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 2, 1, 1, 1, 0 } },
+ { 4, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 5, 3, 1, 1, 1 } },
+ { 5, new int[]{ 3, 2, 0, 0, 1, 0, 0, 0 }, new long[]{ 3, 5, 5, 5, 6, 3, 1, 1 } },
+ {
+ 5, new int[]{ 3, 11, 5, 13, 7, 17, 0, 3, 50, 600, 7000 },
+ new long[]{ 3, 14, 19, 32, 39, 53, 42, 40, 77, 670, 7653 }
+ },
+ };
+ }
+
+ @Test(dataProvider = "simulatedCounterIterations")
+ public void testCounterWithSimulatedRuns(int windowLengthInSlots, int[] incrementsPerIteration,
+ long[] expCountsPerIteration) {
+ // given
+ SlidingWindowCounter<Object> counter = new SlidingWindowCounter<Object>(windowLengthInSlots);
+ int numIterations = incrementsPerIteration.length;
+
+ for (int i = 0; i < numIterations; i++) {
+ int numIncrements = incrementsPerIteration[i];
+ long expCounts = expCountsPerIteration[i];
+ // Objects are absent if they were zero both this iteration
+ // and the last -- if only this one, we need to report zero.
+ boolean expAbsent = ((expCounts == 0) && ((i == 0) || (expCountsPerIteration[i - 1] == 0)));
+
+ // given (for this iteration)
+ for (int j = 0; j < numIncrements; j++) {
+ counter.incrementCount(ANY_OBJECT);
+ }
+
+ // when (for this iteration)
+ Map<Object, Long> counts = counter.getCountsThenAdvanceWindow();
+
+ // then (for this iteration)
+ if (expAbsent) {
+ assertThat(counts).doesNotContainKey(ANY_OBJECT);
+ } else {
+ assertThat(counts.get(ANY_OBJECT)).isEqualTo(expCounts);
+ }
+ }
}
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java
index e4f7dcc..df2ab21 100644
--- a/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java
+++ b/examples/storm-starter/test/jvm/org/apache/storm/starter/tools/SlotBasedCounterTest.java
@@ -1,181 +1,175 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.tools;
+import java.util.Map;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-import java.util.Map;
-
import static org.fest.assertions.api.Assertions.assertThat;
public class SlotBasedCounterTest {
- private static final int ANY_NUM_SLOTS = 1;
- private static final int ANY_SLOT = 0;
- private static final Object ANY_OBJECT = "ANY_OBJECT";
-
- @DataProvider
- public Object[][] illegalNumSlotsData() {
- return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalNumSlotsData")
- public void negativeOrZeroNumSlotsShouldThrowIAE(int numSlots) {
- new SlotBasedCounter<Object>(numSlots);
- }
-
- @DataProvider
- public Object[][] legalNumSlotsData() {
- return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
- }
-
- @Test(dataProvider = "legalNumSlotsData")
- public void positiveNumSlotsShouldBeOk(int numSlots) {
- new SlotBasedCounter<Object>(numSlots);
- }
-
- @Test
- public void newInstanceShouldHaveEmptyCounts() {
- // given
- SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(ANY_NUM_SLOTS);
-
- // when
- Map<Object, Long> counts = counter.getCounts();
-
- // then
- assertThat(counts).isEmpty();
- }
-
- @Test
- public void shouldReturnNonEmptyCountsWhenAtLeastOneObjectWasCounted() {
- // given
- SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(ANY_NUM_SLOTS);
- counter.incrementCount(ANY_OBJECT, ANY_SLOT);
-
- // when
- Map<Object, Long> counts = counter.getCounts();
-
- // then
- assertThat(counts).isNotEmpty();
-
- // additional tests that go beyond what this test is primarily about
- assertThat(counts.size()).isEqualTo(1);
- assertThat(counts.get(ANY_OBJECT)).isEqualTo(1);
- }
-
- @DataProvider
- public Object[][] incrementCountData() {
- return new Object[][]{ { new String[]{ "foo", "bar" }, new int[]{ 3, 2 } } };
- }
-
- @Test(dataProvider = "incrementCountData")
- public void shouldIncrementCount(Object[] objects, int[] expCounts) {
- // given
- SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(ANY_NUM_SLOTS);
-
- // when
- for (int i = 0; i < objects.length; i++) {
- Object obj = objects[i];
- int numIncrements = expCounts[i];
- for (int j = 0; j < numIncrements; j++) {
- counter.incrementCount(obj, ANY_SLOT);
- }
+ private static final int ANY_NUM_SLOTS = 1;
+ private static final int ANY_SLOT = 0;
+ private static final Object ANY_OBJECT = "ANY_OBJECT";
+
+ @DataProvider
+ public Object[][] illegalNumSlotsData() {
+ return new Object[][]{ { -10 }, { -3 }, { -2 }, { -1 }, { 0 } };
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class, dataProvider = "illegalNumSlotsData")
+ public void negativeOrZeroNumSlotsShouldThrowIAE(int numSlots) {
+ new SlotBasedCounter<Object>(numSlots);
+ }
+
+ @DataProvider
+ public Object[][] legalNumSlotsData() {
+ return new Object[][]{ { 1 }, { 2 }, { 3 }, { 20 } };
+ }
+
+ @Test(dataProvider = "legalNumSlotsData")
+ public void positiveNumSlotsShouldBeOk(int numSlots) {
+ new SlotBasedCounter<Object>(numSlots);
+ }
+
+ @Test
+ public void newInstanceShouldHaveEmptyCounts() {
+ // given
+ SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(ANY_NUM_SLOTS);
+
+ // when
+ Map<Object, Long> counts = counter.getCounts();
+
+ // then
+ assertThat(counts).isEmpty();
+ }
+
+ @Test
+ public void shouldReturnNonEmptyCountsWhenAtLeastOneObjectWasCounted() {
+ // given
+ SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(ANY_NUM_SLOTS);
+ counter.incrementCount(ANY_OBJECT, ANY_SLOT);
+
+ // when
+ Map<Object, Long> counts = counter.getCounts();
+
+ // then
+ assertThat(counts).isNotEmpty();
+
+ // additional tests that go beyond what this test is primarily about
+ assertThat(counts.size()).isEqualTo(1);
+ assertThat(counts.get(ANY_OBJECT)).isEqualTo(1);
+ }
+
+ @DataProvider
+ public Object[][] incrementCountData() {
+ return new Object[][]{ { new String[]{ "foo", "bar" }, new int[]{ 3, 2 } } };
+ }
+
+ @Test(dataProvider = "incrementCountData")
+ public void shouldIncrementCount(Object[] objects, int[] expCounts) {
+ // given
+ SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(ANY_NUM_SLOTS);
+
+ // when
+ for (int i = 0; i < objects.length; i++) {
+ Object obj = objects[i];
+ int numIncrements = expCounts[i];
+ for (int j = 0; j < numIncrements; j++) {
+ counter.incrementCount(obj, ANY_SLOT);
+ }
+ }
+
+ // then
+ for (int i = 0; i < objects.length; i++) {
+ assertThat(counter.getCount(objects[i], ANY_SLOT)).isEqualTo(expCounts[i]);
+ }
+ assertThat(counter.getCount("nonexistentObject", ANY_SLOT)).isEqualTo(0);
+ }
+
+ @Test
+ public void shouldReturnZeroForNonexistentObject() {
+ // given
+ SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(ANY_NUM_SLOTS);
+
+ // when
+ counter.incrementCount("somethingElse", ANY_SLOT);
+
+ // then
+ assertThat(counter.getCount("nonexistentObject", ANY_SLOT)).isEqualTo(0);
+ }
+
+ @Test
+ public void shouldIncrementCountOnlyOneSlotAtATime() {
+ // given
+ int numSlots = 3;
+ Object obj = Long.valueOf(10);
+ SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(numSlots);
+
+ // when (empty)
+ // then
+ assertThat(counter.getCount(obj, 0)).isEqualTo(0);
+ assertThat(counter.getCount(obj, 1)).isEqualTo(0);
+ assertThat(counter.getCount(obj, 2)).isEqualTo(0);
+
+ // when
+ counter.incrementCount(obj, 1);
+
+ // then
+ assertThat(counter.getCount(obj, 0)).isEqualTo(0);
+ assertThat(counter.getCount(obj, 1)).isEqualTo(1);
+ assertThat(counter.getCount(obj, 2)).isEqualTo(0);
+ }
+
+ @Test
+ public void wipeSlotShouldSetAllCountsInSlotToZero() {
+ // given
+ SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(ANY_NUM_SLOTS);
+ Object countWasOne = "countWasOne";
+ Object countWasThree = "countWasThree";
+ counter.incrementCount(countWasOne, ANY_SLOT);
+ counter.incrementCount(countWasThree, ANY_SLOT);
+ counter.incrementCount(countWasThree, ANY_SLOT);
+ counter.incrementCount(countWasThree, ANY_SLOT);
+
+ // when
+ counter.wipeSlot(ANY_SLOT);
+
+ // then
+ assertThat(counter.getCount(countWasOne, ANY_SLOT)).isEqualTo(0);
+ assertThat(counter.getCount(countWasThree, ANY_SLOT)).isEqualTo(0);
}
- // then
- for (int i = 0; i < objects.length; i++) {
- assertThat(counter.getCount(objects[i], ANY_SLOT)).isEqualTo(expCounts[i]);
+ @Test
+ public void wipeZerosShouldRemoveAnyObjectsWithZeroTotalCount() {
+ // given
+ SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(2);
+ int wipeSlot = 0;
+ int otherSlot = 1;
+ Object willBeRemoved = "willBeRemoved";
+ Object willContinueToBeTracked = "willContinueToBeTracked";
+ counter.incrementCount(willBeRemoved, wipeSlot);
+ counter.incrementCount(willContinueToBeTracked, wipeSlot);
+ counter.incrementCount(willContinueToBeTracked, otherSlot);
+
+ // when
+ counter.wipeSlot(wipeSlot);
+ counter.wipeZeros();
+
+ // then
+ assertThat(counter.getCounts()).doesNotContainKey(willBeRemoved);
+ assertThat(counter.getCounts()).containsKey(willContinueToBeTracked);
}
- assertThat(counter.getCount("nonexistentObject", ANY_SLOT)).isEqualTo(0);
- }
-
- @Test
- public void shouldReturnZeroForNonexistentObject() {
- // given
- SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(ANY_NUM_SLOTS);
-
- // when
- counter.incrementCount("somethingElse", ANY_SLOT);
-
- // then
- assertThat(counter.getCount("nonexistentObject", ANY_SLOT)).isEqualTo(0);
- }
-
- @Test
- public void shouldIncrementCountOnlyOneSlotAtATime() {
- // given
- int numSlots = 3;
- Object obj = Long.valueOf(10);
- SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(numSlots);
-
- // when (empty)
- // then
- assertThat(counter.getCount(obj, 0)).isEqualTo(0);
- assertThat(counter.getCount(obj, 1)).isEqualTo(0);
- assertThat(counter.getCount(obj, 2)).isEqualTo(0);
-
- // when
- counter.incrementCount(obj, 1);
-
- // then
- assertThat(counter.getCount(obj, 0)).isEqualTo(0);
- assertThat(counter.getCount(obj, 1)).isEqualTo(1);
- assertThat(counter.getCount(obj, 2)).isEqualTo(0);
- }
-
- @Test
- public void wipeSlotShouldSetAllCountsInSlotToZero() {
- // given
- SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(ANY_NUM_SLOTS);
- Object countWasOne = "countWasOne";
- Object countWasThree = "countWasThree";
- counter.incrementCount(countWasOne, ANY_SLOT);
- counter.incrementCount(countWasThree, ANY_SLOT);
- counter.incrementCount(countWasThree, ANY_SLOT);
- counter.incrementCount(countWasThree, ANY_SLOT);
-
- // when
- counter.wipeSlot(ANY_SLOT);
-
- // then
- assertThat(counter.getCount(countWasOne, ANY_SLOT)).isEqualTo(0);
- assertThat(counter.getCount(countWasThree, ANY_SLOT)).isEqualTo(0);
- }
-
- @Test
- public void wipeZerosShouldRemoveAnyObjectsWithZeroTotalCount() {
- // given
- SlotBasedCounter<Object> counter = new SlotBasedCounter<Object>(2);
- int wipeSlot = 0;
- int otherSlot = 1;
- Object willBeRemoved = "willBeRemoved";
- Object willContinueToBeTracked = "willContinueToBeTracked";
- counter.incrementCount(willBeRemoved, wipeSlot);
- counter.incrementCount(willContinueToBeTracked, wipeSlot);
- counter.incrementCount(willContinueToBeTracked, otherSlot);
-
- // when
- counter.wipeSlot(wipeSlot);
- counter.wipeZeros();
-
- // then
- assertThat(counter.getCounts()).doesNotContainKey(willBeRemoved);
- assertThat(counter.getCounts()).containsKey(willContinueToBeTracked);
- }
}
[46/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-kafka
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/OpaqueTridentKafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/OpaqueTridentKafkaSpout.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/OpaqueTridentKafkaSpout.java
index c17c912..c98be42 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/OpaqueTridentKafkaSpout.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/OpaqueTridentKafkaSpout.java
@@ -1,36 +1,30 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.kafka.Partition;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.trident.spout.IOpaquePartitionedTridentSpout;
import org.apache.storm.tuple.Fields;
-import java.util.List;
-import java.util.Map;
-
public class OpaqueTridentKafkaSpout
- implements IOpaquePartitionedTridentSpout<
- List<GlobalPartitionInformation>,
- Partition,
- Map<String, Object>> {
+ implements IOpaquePartitionedTridentSpout<
+ List<GlobalPartitionInformation>,
+ Partition,
+ Map<String, Object>> {
TridentKafkaConfig _config;
@@ -41,17 +35,17 @@ public class OpaqueTridentKafkaSpout
@Override
public Emitter<List<GlobalPartitionInformation>,
- Partition,
- Map<String, Object>> getEmitter(Map<String, Object> conf,
- TopologyContext context) {
+ Partition,
+ Map<String, Object>> getEmitter(Map<String, Object> conf,
+ TopologyContext context) {
return new TridentKafkaEmitter(conf, context, _config, context
- .getStormId()).asOpaqueEmitter();
+ .getStormId()).asOpaqueEmitter();
}
@Override
public IOpaquePartitionedTridentSpout.Coordinator getCoordinator(
- Map<String, Object> conf,
- TopologyContext tc) {
+ Map<String, Object> conf,
+ TopologyContext tc) {
return new org.apache.storm.kafka.trident.Coordinator(conf, _config);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/StaticBrokerReader.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/StaticBrokerReader.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/StaticBrokerReader.java
index ba27651..3c5cc09 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/StaticBrokerReader.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/StaticBrokerReader.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
import java.util.ArrayList;
@@ -24,7 +19,7 @@ import java.util.TreeMap;
public class StaticBrokerReader implements IBrokerReader {
- private Map<String,GlobalPartitionInformation> brokers = new TreeMap<String,GlobalPartitionInformation>();
+ private Map<String, GlobalPartitionInformation> brokers = new TreeMap<String, GlobalPartitionInformation>();
public StaticBrokerReader(String topic, GlobalPartitionInformation partitionInformation) {
this.brokers.put(topic, partitionInformation);
@@ -37,7 +32,7 @@ public class StaticBrokerReader implements IBrokerReader {
}
@Override
- public List<GlobalPartitionInformation> getAllBrokers () {
+ public List<GlobalPartitionInformation> getAllBrokers() {
List<GlobalPartitionInformation> list = new ArrayList<GlobalPartitionInformation>();
list.addAll(brokers.values());
return list;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TransactionalTridentKafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TransactionalTridentKafkaSpout.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TransactionalTridentKafkaSpout.java
index 1042098..7b1d4dd 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TransactionalTridentKafkaSpout.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TransactionalTridentKafkaSpout.java
@@ -1,29 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
+import java.util.Map;
import org.apache.storm.kafka.Partition;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.trident.spout.IPartitionedTridentSpout;
import org.apache.storm.tuple.Fields;
-import java.util.Map;
-
public class TransactionalTridentKafkaSpout implements IPartitionedTridentSpout<GlobalPartitionInformation, Partition, Map> {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaConfig.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaConfig.java
index b225e9a..3dac221 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaConfig.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaConfig.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
import org.apache.storm.kafka.BrokerHosts;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaEmitter.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaEmitter.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaEmitter.java
index 3333c2c..cb00579 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaEmitter.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaEmitter.java
@@ -1,24 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
import com.google.common.collect.ImmutableMap;
-
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import kafka.javaapi.consumer.SimpleConsumer;
+import kafka.javaapi.message.ByteBufferMessageSet;
+import kafka.message.Message;
+import kafka.message.MessageAndOffset;
import org.apache.storm.Config;
import org.apache.storm.kafka.DynamicPartitionConnections;
import org.apache.storm.kafka.FailedFetchException;
@@ -38,17 +41,6 @@ import org.apache.storm.trident.topology.TransactionAttempt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import kafka.javaapi.consumer.SimpleConsumer;
-import kafka.javaapi.message.ByteBufferMessageSet;
-import kafka.message.Message;
-import kafka.message.MessageAndOffset;
-
public class TridentKafkaEmitter {
private static final Logger LOG = LoggerFactory.getLogger(TridentKafkaEmitter.class);
@@ -74,10 +66,10 @@ public class TridentKafkaEmitter {
private Map<String, Object> failFastEmitNewPartitionBatch(
- final TransactionAttempt attempt,
- TridentCollector collector,
- Partition partition,
- Map<String, Object> lastMeta) {
+ final TransactionAttempt attempt,
+ TridentCollector collector,
+ Partition partition,
+ Map<String, Object> lastMeta) {
SimpleConsumer consumer = _connections.register(partition);
Map<String, Object> ret = doEmitNewPartitionBatch(consumer, partition, collector, lastMeta, attempt);
Long offset = (Long) ret.get("offset");
@@ -86,7 +78,8 @@ public class TridentKafkaEmitter {
return ret;
}
- private Map<String, Object> emitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map<String, Object> lastMeta) {
+ private Map<String, Object> emitNewPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition,
+ Map<String, Object> lastMeta) {
try {
return failFastEmitNewPartitionBatch(attempt, collector, partition, lastMeta);
} catch (FailedFetchException e) {
@@ -107,16 +100,16 @@ public class TridentKafkaEmitter {
}
private Map<String, Object> doEmitNewPartitionBatch(SimpleConsumer consumer,
- Partition partition,
- TridentCollector collector,
- Map<String, Object> lastMeta,
- TransactionAttempt attempt) {
+ Partition partition,
+ TridentCollector collector,
+ Map<String, Object> lastMeta,
+ TransactionAttempt attempt) {
LOG.debug("Emitting new partition batch - [transaction = {}], [lastMeta = {}]", attempt, lastMeta);
long offset;
if (lastMeta != null) {
String lastInstanceId = null;
Map<String, Object> lastTopoMeta = (Map<String, Object>)
- lastMeta.get("topology");
+ lastMeta.get("topology");
if (lastTopoMeta != null) {
lastInstanceId = (String) lastTopoMeta.get("id");
}
@@ -170,7 +163,8 @@ public class TridentKafkaEmitter {
/**
* re-emit the batch described by the meta data provided
*/
- private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map<String, Object> meta) {
+ private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition,
+ Map<String, Object> meta) {
LOG.info("re-emitting batch, attempt " + attempt);
String instanceId = (String) meta.get("instanceId");
if (!_config.ignoreZkOffsets || instanceId.equals(_topologyInstanceId)) {
@@ -241,7 +235,8 @@ public class TridentKafkaEmitter {
* for defining the parameters of the next batch.
*/
@Override
- public Map<String, Object> emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map<String, Object> map) {
+ public Map<String, Object> emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector,
+ Partition partition, Map<String, Object> map) {
return emitNewPartitionBatch(transactionAttempt, tridentCollector, partition, map);
}
@@ -270,7 +265,8 @@ public class TridentKafkaEmitter {
* Return the metadata that can be used to reconstruct this partition/batch in the future.
*/
@Override
- public Map<String, Object> emitPartitionBatchNew(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map<String, Object> map) {
+ public Map<String, Object> emitPartitionBatchNew(TransactionAttempt transactionAttempt, TridentCollector tridentCollector,
+ Partition partition, Map<String, Object> map) {
return failFastEmitNewPartitionBatch(transactionAttempt, tridentCollector, partition, map);
}
@@ -279,7 +275,8 @@ public class TridentKafkaEmitter {
* the metadata created when it was first emitted.
*/
@Override
- public void emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition, Map<String, Object> map) {
+ public void emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector tridentCollector, Partition partition,
+ Map<String, Object> map) {
reEmitPartitionBatch(transactionAttempt, tridentCollector, partition, map);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaState.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaState.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaState.java
index eb6737a..71b2cb1 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaState.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaState.java
@@ -1,41 +1,35 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.topology.FailedException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
import org.apache.commons.lang.Validate;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.storm.kafka.trident.mapper.TridentTupleToKafkaMapper;
import org.apache.storm.kafka.trident.selector.KafkaTopicSelector;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.topology.FailedException;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.tuple.TridentTuple;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TridentKafkaState implements State {
private static final Logger LOG = LoggerFactory.getLogger(TridentKafkaState.class);
@@ -79,9 +73,10 @@ public class TridentKafkaState implements State {
for (TridentTuple tuple : tuples) {
topic = topicSelector.getTopic(tuple);
- if(topic != null) {
+ if (topic != null) {
Future<RecordMetadata> result = producer.send(new ProducerRecord(topic,
- mapper.getKeyFromTuple(tuple), mapper.getMessageFromTuple(tuple)));
+ mapper.getKeyFromTuple(tuple),
+ mapper.getMessageFromTuple(tuple)));
futures.add(result);
} else {
LOG.warn("skipping key = " + mapper.getKeyFromTuple(tuple) + ", topic selector returned null.");
@@ -97,9 +92,9 @@ public class TridentKafkaState implements State {
}
}
- if(exceptions.size() > 0){
- String errorMsg = "Could not retrieve result for messages " + tuples + " from topic = " + topic
- + " because of the following exceptions: \n";
+ if (exceptions.size() > 0) {
+ String errorMsg = "Could not retrieve result for messages " + tuples + " from topic = " + topic
+ + " because of the following exceptions: \n";
for (ExecutionException exception : exceptions) {
errorMsg = errorMsg + exception.getMessage() + "\n";
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaStateFactory.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaStateFactory.java
index 0bf21ab..5b66fd8 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaStateFactory.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaStateFactory.java
@@ -1,32 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
-import org.apache.storm.task.IMetricsContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.Map;
+import java.util.Properties;
import org.apache.storm.kafka.trident.mapper.TridentTupleToKafkaMapper;
import org.apache.storm.kafka.trident.selector.KafkaTopicSelector;
+import org.apache.storm.task.IMetricsContext;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.state.StateFactory;
-
-import java.util.Map;
-import java.util.Properties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TridentKafkaStateFactory implements StateFactory {
@@ -55,8 +49,8 @@ public class TridentKafkaStateFactory implements StateFactory {
public State makeState(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
LOG.info("makeState(partitonIndex={}, numpartitions={}", partitionIndex, numPartitions);
TridentKafkaState state = new TridentKafkaState()
- .withKafkaTopicSelector(this.topicSelector)
- .withTridentTupleToKafkaMapper(this.mapper);
+ .withKafkaTopicSelector(this.topicSelector)
+ .withTridentTupleToKafkaMapper(this.mapper);
state.prepare(producerProperties);
return state;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaUpdater.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaUpdater.java
index 7a905ab..1100b66 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaUpdater.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/TridentKafkaUpdater.java
@@ -1,28 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident;
+import java.util.List;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseStateUpdater;
import org.apache.storm.trident.tuple.TridentTuple;
-import java.util.List;
-
public class TridentKafkaUpdater extends BaseStateUpdater<TridentKafkaState> {
@Override
public void updateState(TridentKafkaState state, List<TridentTuple> tuples, TridentCollector collector) {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/ZkBrokerReader.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/ZkBrokerReader.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/ZkBrokerReader.java
index 00758a6..d40256e 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/ZkBrokerReader.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/ZkBrokerReader.java
@@ -1,84 +1,79 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka.trident;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.storm.kafka.DynamicBrokersReader;
-import org.apache.storm.kafka.ZkHosts;
+package org.apache.storm.kafka.trident;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.storm.kafka.DynamicBrokersReader;
+import org.apache.storm.kafka.ZkHosts;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ZkBrokerReader implements IBrokerReader {
- private static final Logger LOG = LoggerFactory.getLogger(ZkBrokerReader.class);
+ private static final Logger LOG = LoggerFactory.getLogger(ZkBrokerReader.class);
- List<GlobalPartitionInformation> cachedBrokers = new ArrayList<GlobalPartitionInformation>();
- DynamicBrokersReader reader;
- long lastRefreshTimeMs;
+ List<GlobalPartitionInformation> cachedBrokers = new ArrayList<GlobalPartitionInformation>();
+ DynamicBrokersReader reader;
+ long lastRefreshTimeMs;
- long refreshMillis;
+ long refreshMillis;
- public ZkBrokerReader(Map<String, Object> conf, String topic, ZkHosts hosts) {
- try {
- reader = new DynamicBrokersReader(conf, hosts.brokerZkStr, hosts.brokerZkPath, topic);
- cachedBrokers = reader.getBrokerInfo();
- lastRefreshTimeMs = System.currentTimeMillis();
- refreshMillis = hosts.refreshFreqSecs * 1000L;
- } catch (java.net.SocketTimeoutException e) {
- LOG.warn("Failed to update brokers", e);
- }
+ public ZkBrokerReader(Map<String, Object> conf, String topic, ZkHosts hosts) {
+ try {
+ reader = new DynamicBrokersReader(conf, hosts.brokerZkStr, hosts.brokerZkPath, topic);
+ cachedBrokers = reader.getBrokerInfo();
+ lastRefreshTimeMs = System.currentTimeMillis();
+ refreshMillis = hosts.refreshFreqSecs * 1000L;
+ } catch (java.net.SocketTimeoutException e) {
+ LOG.warn("Failed to update brokers", e);
+ }
+
+ }
- }
+ private void refresh() {
+ long currTime = System.currentTimeMillis();
+ if (currTime > lastRefreshTimeMs + refreshMillis) {
+ try {
+ LOG.info("brokers need refreshing because " + refreshMillis + "ms have expired");
+ cachedBrokers = reader.getBrokerInfo();
+ lastRefreshTimeMs = currTime;
+ } catch (java.net.SocketTimeoutException e) {
+ LOG.warn("Failed to update brokers", e);
+ }
+ }
+ }
- private void refresh() {
- long currTime = System.currentTimeMillis();
- if (currTime > lastRefreshTimeMs + refreshMillis) {
- try {
- LOG.info("brokers need refreshing because " + refreshMillis + "ms have expired");
- cachedBrokers = reader.getBrokerInfo();
- lastRefreshTimeMs = currTime;
- } catch (java.net.SocketTimeoutException e) {
- LOG.warn("Failed to update brokers", e);
- }
- }
- }
- @Override
- public GlobalPartitionInformation getBrokerForTopic(String topic) {
- refresh();
- for(GlobalPartitionInformation partitionInformation : cachedBrokers) {
+ @Override
+ public GlobalPartitionInformation getBrokerForTopic(String topic) {
+ refresh();
+ for (GlobalPartitionInformation partitionInformation : cachedBrokers) {
if (partitionInformation.topic.equals(topic)) return partitionInformation;
}
- return null;
- }
+ return null;
+ }
- @Override
- public List<GlobalPartitionInformation> getAllBrokers() {
- refresh();
- return cachedBrokers;
- }
+ @Override
+ public List<GlobalPartitionInformation> getAllBrokers() {
+ refresh();
+ return cachedBrokers;
+ }
- @Override
- public void close() {
- reader.close();
- }
+ @Override
+ public void close() {
+ reader.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/mapper/FieldNameBasedTupleToKafkaMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/mapper/FieldNameBasedTupleToKafkaMapper.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/mapper/FieldNameBasedTupleToKafkaMapper.java
index 2d04971..01e3eca 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/mapper/FieldNameBasedTupleToKafkaMapper.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/mapper/FieldNameBasedTupleToKafkaMapper.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident.mapper;
import org.apache.storm.trident.tuple.TridentTuple;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/mapper/TridentTupleToKafkaMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/mapper/TridentTupleToKafkaMapper.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/mapper/TridentTupleToKafkaMapper.java
index 28c6c89..4a522d6 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/mapper/TridentTupleToKafkaMapper.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/mapper/TridentTupleToKafkaMapper.java
@@ -1,28 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka.trident.mapper;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.trident.tuple.TridentTuple;
+package org.apache.storm.kafka.trident.mapper;
import java.io.Serializable;
+import org.apache.storm.trident.tuple.TridentTuple;
-public interface TridentTupleToKafkaMapper<K,V> extends Serializable {
+public interface TridentTupleToKafkaMapper<K, V> extends Serializable {
K getKeyFromTuple(TridentTuple tuple);
+
V getMessageFromTuple(TridentTuple tuple);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/selector/DefaultTopicSelector.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/selector/DefaultTopicSelector.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/selector/DefaultTopicSelector.java
index 7ae49a3..93b5566 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/selector/DefaultTopicSelector.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/selector/DefaultTopicSelector.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.trident.selector;
import org.apache.storm.trident.tuple.TridentTuple;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/selector/KafkaTopicSelector.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/selector/KafkaTopicSelector.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/selector/KafkaTopicSelector.java
index 012a6c7..6de3921 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/selector/KafkaTopicSelector.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/trident/selector/KafkaTopicSelector.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka.trident.selector;
-import org.apache.storm.trident.tuple.TridentTuple;
+package org.apache.storm.kafka.trident.selector;
import java.io.Serializable;
+import org.apache.storm.trident.tuple.TridentTuple;
public interface KafkaTopicSelector extends Serializable {
String getTopic(TridentTuple tuple);
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/DynamicBrokersReaderTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/DynamicBrokersReaderTest.java b/external/storm-kafka/src/test/org/apache/storm/kafka/DynamicBrokersReaderTest.java
index 46cc60d..a6bb61c 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/DynamicBrokersReaderTest.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/DynamicBrokersReaderTest.java
@@ -1,40 +1,33 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
-import org.apache.storm.Config;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.curator.test.TestingServer;
import org.apache.curator.utils.ZKPaths;
+import org.apache.storm.Config;
+import org.apache.storm.kafka.trident.GlobalPartitionInformation;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.apache.storm.kafka.trident.GlobalPartitionInformation;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
/**
* Date: 16/05/2013
@@ -66,7 +59,7 @@ public class DynamicBrokersReaderTest {
Map<String, Object> conf2 = new HashMap<>();
conf2.putAll(conf);
- conf2.put("kafka.topic.wildcard.match",true);
+ conf2.put("kafka.topic.wildcard.match", true);
wildCardBrokerReader = new DynamicBrokersReader(conf2, connectionString, masterPath, "^test.*$");
zookeeper.start();
@@ -114,8 +107,8 @@ public class DynamicBrokersReaderTest {
}
- private GlobalPartitionInformation getByTopic(List<GlobalPartitionInformation> partitions, String topic){
- for(GlobalPartitionInformation partitionInformation : partitions) {
+ private GlobalPartitionInformation getByTopic(List<GlobalPartitionInformation> partitions, String topic) {
+ for (GlobalPartitionInformation partitionInformation : partitions) {
if (partitionInformation.topic.equals(topic)) return partitionInformation;
}
return null;
@@ -242,7 +235,7 @@ public class DynamicBrokersReaderTest {
String connectionString = server.getConnectString();
Map<String, Object> conf = new HashMap<>();
conf.put(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT, 1000);
-// conf.put(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT, 1000);
+ // conf.put(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT, 1000);
conf.put(Config.STORM_ZOOKEEPER_RETRY_TIMES, 4);
conf.put(Config.STORM_ZOOKEEPER_RETRY_INTERVAL, 5);
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java b/external/storm-kafka/src/test/org/apache/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java
index 049fce7..5fcee28 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/ExponentialBackoffMsgRetryManagerTest.java
@@ -1,28 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka;
-import org.junit.Test;
+package org.apache.storm.kafka;
import org.apache.storm.utils.Time;
import org.apache.storm.utils.Time.SimulatedTime;
import org.junit.After;
import org.junit.Before;
+import org.junit.Test;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -52,8 +47,8 @@ public class ExponentialBackoffMsgRetryManagerTest {
@Test
public void testImmediateRetry() throws Exception {
-
-
+
+
ExponentialBackoffMsgRetryManager manager = buildExponentialBackoffMsgRetryManager(0, 0d, 0, Integer.MAX_VALUE);
manager.failed(TEST_OFFSET);
Long next = manager.nextFailedMessageToRetry();
@@ -124,22 +119,22 @@ public class ExponentialBackoffMsgRetryManagerTest {
// so TEST_OFFSET2 should come first
Time.advanceTime(initial * 2);
- assertTrue("message "+TEST_OFFSET+"should be ready for retry", manager.shouldReEmitMsg(TEST_OFFSET));
- assertTrue("message "+TEST_OFFSET2+"should be ready for retry", manager.shouldReEmitMsg(TEST_OFFSET2));
+ assertTrue("message " + TEST_OFFSET + "should be ready for retry", manager.shouldReEmitMsg(TEST_OFFSET));
+ assertTrue("message " + TEST_OFFSET2 + "should be ready for retry", manager.shouldReEmitMsg(TEST_OFFSET2));
Long next = manager.nextFailedMessageToRetry();
- assertEquals("expect first message to retry is "+TEST_OFFSET2, TEST_OFFSET2, next);
+ assertEquals("expect first message to retry is " + TEST_OFFSET2, TEST_OFFSET2, next);
Time.advanceTime(initial);
// haven't retried yet, so first should still be TEST_OFFSET2
next = manager.nextFailedMessageToRetry();
- assertEquals("expect first message to retry is "+TEST_OFFSET2, TEST_OFFSET2, next);
+ assertEquals("expect first message to retry is " + TEST_OFFSET2, TEST_OFFSET2, next);
manager.retryStarted(next);
// now it should be TEST_OFFSET
next = manager.nextFailedMessageToRetry();
- assertEquals("expect message to retry is now "+TEST_OFFSET, TEST_OFFSET, next);
+ assertEquals("expect message to retry is now " + TEST_OFFSET, TEST_OFFSET, next);
manager.retryStarted(next);
// now none left
@@ -230,14 +225,14 @@ public class ExponentialBackoffMsgRetryManagerTest {
assertEquals("expect test offset next available for retry", TEST_OFFSET, next);
assertTrue("message should be ready for retry", manager.shouldReEmitMsg(TEST_OFFSET));
}
-
+
@Test
public void testClearInvalidMessages() throws Exception {
ExponentialBackoffMsgRetryManager manager = buildExponentialBackoffMsgRetryManager(0, 0d, 0, Integer.MAX_VALUE);
manager.failed(TEST_OFFSET);
manager.failed(TEST_OFFSET2);
manager.failed(TEST_OFFSET3);
-
+
assertTrue("message should be ready for retry", manager.shouldReEmitMsg(TEST_OFFSET));
assertTrue("message should be ready for retry", manager.shouldReEmitMsg(TEST_OFFSET2));
assertTrue("message should be ready for retry", manager.shouldReEmitMsg(TEST_OFFSET3));
@@ -246,7 +241,7 @@ public class ExponentialBackoffMsgRetryManagerTest {
Long next = manager.nextFailedMessageToRetry();
assertEquals("expect test offset next available for retry", TEST_OFFSET3, next);
-
+
manager.acked(TEST_OFFSET3);
next = manager.nextFailedMessageToRetry();
assertNull("expect no message ready after acked", next);
@@ -267,8 +262,8 @@ public class ExponentialBackoffMsgRetryManagerTest {
assertFalse(manager.retryFurther(TEST_OFFSET));
}
-
- private ExponentialBackoffMsgRetryManager buildExponentialBackoffMsgRetryManager(long retryInitialDelayMs,
+
+ private ExponentialBackoffMsgRetryManager buildExponentialBackoffMsgRetryManager(long retryInitialDelayMs,
double retryDelayMultiplier,
long retryDelayMaxMs,
int retryLimit) {
@@ -276,7 +271,7 @@ public class ExponentialBackoffMsgRetryManagerTest {
spoutConfig.retryInitialDelayMs = retryInitialDelayMs;
spoutConfig.retryDelayMultiplier = retryDelayMultiplier;
spoutConfig.retryDelayMaxMs = retryDelayMaxMs;
- spoutConfig.retryLimit = retryLimit;
+ spoutConfig.retryLimit = retryLimit;
ExponentialBackoffMsgRetryManager exponentialBackoffMsgRetryManager = new ExponentialBackoffMsgRetryManager();
exponentialBackoffMsgRetryManager.prepare(spoutConfig, null);
return exponentialBackoffMsgRetryManager;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaErrorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaErrorTest.java b/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaErrorTest.java
index e38bc1e..ad793ed 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaErrorTest.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaErrorTest.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaTestBroker.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaTestBroker.java b/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaTestBroker.java
index 0952764..f31386a 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaTestBroker.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaTestBroker.java
@@ -1,22 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
import kafka.admin.AdminUtils;
import kafka.api.PartitionMetadata;
import kafka.api.TopicMetadata;
@@ -34,11 +33,6 @@ import org.apache.curator.test.InstanceSpec;
import org.apache.curator.test.TestingServer;
import scala.collection.JavaConversions;
-import java.io.File;
-import java.io.IOException;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
/**
* Date: 11/01/2014
* Time: 13:15
@@ -161,6 +155,7 @@ public class KafkaTestBroker {
public int getPort() {
return port;
}
+
public void shutdown() {
if (kafka != null) {
kafka.shutdown();
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaUtilsTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaUtilsTest.java b/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaUtilsTest.java
index 1bd989f..3c2c0d9 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaUtilsTest.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/KafkaUtilsTest.java
@@ -1,38 +1,29 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.kafka;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
+package org.apache.storm.kafka;
+import com.google.common.collect.ImmutableMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
-
import kafka.api.OffsetRequest;
import kafka.javaapi.consumer.SimpleConsumer;
import kafka.javaapi.message.ByteBufferMessageSet;
import kafka.message.MessageAndOffset;
-
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.storm.kafka.trident.GlobalPartitionInformation;
+import org.apache.storm.spout.SchemeAsMultiScheme;
import org.apache.storm.utils.Utils;
import org.junit.After;
import org.junit.Assert;
@@ -42,13 +33,14 @@ import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.storm.kafka.trident.GlobalPartitionInformation;
-import org.apache.storm.spout.SchemeAsMultiScheme;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
-import com.google.common.collect.ImmutableMap;
public class KafkaUtilsTest {
- private String TEST_TOPIC = "testTopic";
private static final Logger LOG = LoggerFactory.getLogger(KafkaUtilsTest.class);
+ private String TEST_TOPIC = "testTopic";
private KafkaTestBroker broker;
private SimpleConsumer simpleConsumer;
private KafkaConfig config;
@@ -73,7 +65,8 @@ public class KafkaUtilsTest {
@Test(expected = FailedFetchException.class)
public void topicDoesNotExist() throws Exception {
- KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), TEST_TOPIC, 0), 0);
+ KafkaUtils
+ .fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), TEST_TOPIC, 0), 0);
}
@Test(expected = FailedFetchException.class)
@@ -82,7 +75,9 @@ public class KafkaUtilsTest {
broker.shutdown();
SimpleConsumer simpleConsumer = new SimpleConsumer("localhost", port, 100, 1024, "testClient");
try {
- KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), TEST_TOPIC, 0), OffsetRequest.LatestTime());
+ KafkaUtils
+ .fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), TEST_TOPIC, 0),
+ OffsetRequest.LatestTime());
} finally {
simpleConsumer.close();
}
@@ -94,7 +89,9 @@ public class KafkaUtilsTest {
createTopicAndSendMessage(value);
long offset = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, OffsetRequest.LatestTime()) - 1;
ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(config, simpleConsumer,
- new Partition(Broker.fromString(broker.getBrokerConnectionString()), TEST_TOPIC, 0), offset);
+ new Partition(
+ Broker.fromString(broker.getBrokerConnectionString()),
+ TEST_TOPIC, 0), offset);
String message = new String(Utils.toByteArray(messageAndOffsets.iterator().next().message().payload()));
assertThat(message, is(equalTo(value)));
}
@@ -103,7 +100,7 @@ public class KafkaUtilsTest {
public void fetchMessagesWithInvalidOffsetAndDefaultHandlingDisabled() throws Exception {
config.useStartOffsetTimeIfOffsetOutOfRange = false;
KafkaUtils.fetchMessages(config, simpleConsumer,
- new Partition(Broker.fromString(broker.getBrokerConnectionString()), TEST_TOPIC, 0), -99);
+ new Partition(Broker.fromString(broker.getBrokerConnectionString()), TEST_TOPIC, 0), -99);
}
@Test(expected = TopicOffsetOutOfRangeException.class)
@@ -112,7 +109,7 @@ public class KafkaUtilsTest {
String value = "test";
createTopicAndSendMessage(value);
KafkaUtils.fetchMessages(config, simpleConsumer,
- new Partition(Broker.fromString(broker.getBrokerConnectionString()), "newTopic", 0), -99);
+ new Partition(Broker.fromString(broker.getBrokerConnectionString()), "newTopic", 0), -99);
}
@Test
@@ -187,21 +184,21 @@ public class KafkaUtilsTest {
assertEquals(value, lists.iterator().next().get(0));
}
}
-
+
@Test
public void generateTuplesWithMessageAndMetadataScheme() {
String value = "value";
Partition mockPartition = Mockito.mock(Partition.class);
mockPartition.partition = 0;
long offset = 0L;
-
+
MessageMetadataSchemeAsMultiScheme scheme = new MessageMetadataSchemeAsMultiScheme(new StringMessageAndMetadataScheme());
-
+
createTopicAndSendMessage(null, value);
ByteBufferMessageSet messageAndOffsets = getLastMessage();
for (MessageAndOffset msg : messageAndOffsets) {
Iterable<List<Object>> lists = KafkaUtils.generateTuples(scheme, msg.message(), mockPartition, offset);
- List<Object> values = lists.iterator().next();
+ List<Object> values = lists.iterator().next();
assertEquals("Message is incorrect", value, values.get(0));
assertEquals("Partition is incorrect", mockPartition.partition, values.get(1));
assertEquals("Offset is incorrect", offset, values.get(2));
@@ -210,12 +207,14 @@ public class KafkaUtilsTest {
private ByteBufferMessageSet getLastMessage() {
long offsetOfLastMessage = KafkaUtils.getOffset(simpleConsumer, config.topic, 0, OffsetRequest.LatestTime()) - 1;
- return KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), TEST_TOPIC, 0), offsetOfLastMessage);
+ return KafkaUtils
+ .fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), TEST_TOPIC, 0),
+ offsetOfLastMessage);
}
private void runGetValueOnlyTuplesTest() {
String value = "value";
-
+
createTopicAndSendMessage(null, value);
ByteBufferMessageSet messageAndOffsets = getLastMessage();
for (MessageAndOffset msg : messageAndOffsets) {
@@ -264,13 +263,13 @@ public class KafkaUtilsTest {
public void assignAllPartitionsToOneTask() {
runPartitionToTaskMappingTest(32, 32);
}
-
+
public void runPartitionToTaskMappingTest(int numPartitions, int partitionsPerTask) {
GlobalPartitionInformation globalPartitionInformation = TestUtils.buildPartitionInfo(numPartitions);
List<GlobalPartitionInformation> partitions = new ArrayList<GlobalPartitionInformation>();
partitions.add(globalPartitionInformation);
int numTasks = numPartitions / partitionsPerTask;
- for (int i = 0 ; i < numTasks ; i++) {
+ for (int i = 0; i < numTasks; i++) {
assertEquals(partitionsPerTask, KafkaUtils.calculatePartitionsForTask(partitions, numTasks, i, i).size());
}
}
@@ -285,7 +284,7 @@ public class KafkaUtilsTest {
assertEquals(0, KafkaUtils.calculatePartitionsForTask(partitions, numTasks, 1, 1).size());
}
- @Test (expected = IllegalArgumentException.class )
+ @Test(expected = IllegalArgumentException.class)
public void assignInvalidTask() {
GlobalPartitionInformation globalPartitionInformation = new GlobalPartitionInformation(TEST_TOPIC);
List<GlobalPartitionInformation> partitions = new ArrayList<GlobalPartitionInformation>();
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/PartitionManagerTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/PartitionManagerTest.java b/external/storm-kafka/src/test/org/apache/storm/kafka/PartitionManagerTest.java
index 805913d..a2824fb 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/PartitionManagerTest.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/PartitionManagerTest.java
@@ -1,22 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.storm.Config;
@@ -30,13 +31,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-
public class PartitionManagerTest {
private static final String TOPIC_NAME = "testTopic";
@@ -128,7 +122,7 @@ public class PartitionManagerTest {
PartitionManager partitionManager = partitionManagers.get(0);
- for (int i=0; i < 5; i++) {
+ for (int i = 0; i < 5; i++) {
sendMessage("message-" + i);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/StringKeyValueSchemeTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/StringKeyValueSchemeTest.java b/external/storm-kafka/src/test/org/apache/storm/kafka/StringKeyValueSchemeTest.java
index 7e5ff00..9c749fe 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/StringKeyValueSchemeTest.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/StringKeyValueSchemeTest.java
@@ -1,29 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
-import org.apache.storm.tuple.Fields;
import com.google.common.collect.ImmutableMap;
-import org.junit.Test;
-
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.Collections;
+import org.apache.storm.tuple.Fields;
+import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -32,6 +26,10 @@ public class StringKeyValueSchemeTest {
private StringKeyValueScheme scheme = new StringKeyValueScheme();
+ private static ByteBuffer wrapString(String s) {
+ return ByteBuffer.wrap(s.getBytes(Charset.defaultCharset()));
+ }
+
@Test
public void testDeserialize() throws Exception {
assertEquals(Collections.singletonList("test"), scheme.deserialize(wrapString("test")));
@@ -47,16 +45,12 @@ public class StringKeyValueSchemeTest {
@Test
public void testDeserializeWithNullKeyAndValue() throws Exception {
assertEquals(Collections.singletonList("test"),
- scheme.deserializeKeyAndValue(null, wrapString("test")));
+ scheme.deserializeKeyAndValue(null, wrapString("test")));
}
@Test
public void testDeserializeWithKeyAndValue() throws Exception {
assertEquals(Collections.singletonList(ImmutableMap.of("key", "test")),
- scheme.deserializeKeyAndValue(wrapString("key"), wrapString("test")));
- }
-
- private static ByteBuffer wrapString(String s) {
- return ByteBuffer.wrap(s.getBytes(Charset.defaultCharset()));
+ scheme.deserializeKeyAndValue(wrapString("key"), wrapString("test")));
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/TestStringScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/TestStringScheme.java b/external/storm-kafka/src/test/org/apache/storm/kafka/TestStringScheme.java
index 23944ab..ac94e9d 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/TestStringScheme.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/TestStringScheme.java
@@ -15,26 +15,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.kafka;
-import org.junit.Test;
+package org.apache.storm.kafka;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import org.junit.Test;
import static org.junit.Assert.assertEquals;
public class TestStringScheme {
- @Test
- public void testDeserializeString() {
- String s = "foo";
- byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
- ByteBuffer direct = ByteBuffer.allocateDirect(bytes.length);
- direct.put(bytes);
- direct.flip();
- String s1 = StringScheme.deserializeString(ByteBuffer.wrap(bytes));
- String s2 = StringScheme.deserializeString(direct);
- assertEquals(s, s1);
- assertEquals(s, s2);
- }
+ @Test
+ public void testDeserializeString() {
+ String s = "foo";
+ byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
+ ByteBuffer direct = ByteBuffer.allocateDirect(bytes.length);
+ direct.put(bytes);
+ direct.flip();
+ String s1 = StringScheme.deserializeString(ByteBuffer.wrap(bytes));
+ String s2 = StringScheme.deserializeString(direct);
+ assertEquals(s, s1);
+ assertEquals(s, s2);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/TestUtils.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/TestUtils.java b/external/storm-kafka/src/test/org/apache/storm/kafka/TestUtils.java
index c7ba674..921df3c 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/TestUtils.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/TestUtils.java
@@ -1,32 +1,28 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
-import org.apache.storm.utils.Utils;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
import kafka.api.OffsetRequest;
import kafka.javaapi.consumer.SimpleConsumer;
import kafka.javaapi.message.ByteBufferMessageSet;
import kafka.message.Message;
import kafka.message.MessageAndOffset;
import org.apache.storm.kafka.trident.GlobalPartitionInformation;
-
-import java.nio.ByteBuffer;
-import java.util.*;
+import org.apache.storm.utils.Utils;
import static org.junit.Assert.assertEquals;
@@ -83,7 +79,9 @@ public class TestUtils {
public static boolean verifyMessage(String key, String message, KafkaTestBroker broker, SimpleConsumer simpleConsumer) {
long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, TestUtils.TOPIC, 0, OffsetRequest.LatestTime()) - 1;
ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(TestUtils.getKafkaConfig(broker), simpleConsumer,
- new Partition(Broker.fromString(broker.getBrokerConnectionString()),TestUtils.TOPIC, 0), lastMessageOffset);
+ new Partition(
+ Broker.fromString(broker.getBrokerConnectionString()),
+ TestUtils.TOPIC, 0), lastMessageOffset);
MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
Message kafkaMessage = messageAndOffset.message();
ByteBuffer messageKeyBuffer = kafkaMessage.key();
[35/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-redis
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/Options.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/Options.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/Options.java
index 3442c46..a65790a 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/Options.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/Options.java
@@ -1,28 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.trident.state;
+import java.io.Serializable;
import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
import org.apache.storm.trident.state.Serializer;
-import java.io.Serializable;
-
/**
* Options of State.<br/>
* It's a data structure (whole things are public) and you can access and modify all fields.
@@ -30,7 +23,8 @@ import java.io.Serializable;
* @param <T> value's type class
*/
public class Options<T> implements Serializable {
- private static final RedisDataTypeDescription DEFAULT_REDIS_DATATYPE = new RedisDataTypeDescription(RedisDataTypeDescription.RedisDataType.STRING);
+ private static final RedisDataTypeDescription DEFAULT_REDIS_DATATYPE =
+ new RedisDataTypeDescription(RedisDataTypeDescription.RedisDataType.STRING);
public int localCacheSize = 1000;
public String globalKey = "$REDIS-MAP-STATE-GLOBAL";
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterMapState.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterMapState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterMapState.java
index ecae18d..c43a18d 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterMapState.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterMapState.java
@@ -1,29 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.trident.state;
-import org.apache.storm.task.IMetricsContext;
-import org.apache.storm.tuple.Values;
import com.google.common.collect.Lists;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.redis.common.config.JedisClusterConfig;
import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
-import redis.clients.jedis.JedisCluster;
+import org.apache.storm.task.IMetricsContext;
import org.apache.storm.trident.state.OpaqueValue;
import org.apache.storm.trident.state.Serializer;
import org.apache.storm.trident.state.State;
@@ -36,9 +30,8 @@ import org.apache.storm.trident.state.map.NonTransactionalMap;
import org.apache.storm.trident.state.map.OpaqueMap;
import org.apache.storm.trident.state.map.SnapshottableMap;
import org.apache.storm.trident.state.map.TransactionalMap;
-
-import java.util.List;
-import java.util.Map;
+import org.apache.storm.tuple.Values;
+import redis.clients.jedis.JedisCluster;
/**
* IBackingMap implementation for Redis Cluster environment.
@@ -47,6 +40,27 @@ import java.util.Map;
* @see AbstractRedisMapState
*/
public class RedisClusterMapState<T> extends AbstractRedisMapState<T> {
+ private JedisCluster jedisCluster;
+ private Options options;
+ private Serializer serializer;
+ private KeyFactory keyFactory;
+
+ /**
+ * Constructor
+ *
+ * @param jedisCluster JedisCluster
+ * @param options options of State
+ * @param serializer Serializer
+ * @param keyFactory KeyFactory
+ */
+ public RedisClusterMapState(JedisCluster jedisCluster, Options options,
+ Serializer<T> serializer, KeyFactory keyFactory) {
+ this.jedisCluster = jedisCluster;
+ this.options = options;
+ this.serializer = serializer;
+ this.keyFactory = keyFactory;
+ }
+
/**
* Provides StateFactory for opaque transactional.
*
@@ -189,98 +203,6 @@ public class RedisClusterMapState<T> extends AbstractRedisMapState<T> {
}
/**
- * RedisClusterMapState.Factory provides Redis Cluster environment version of StateFactory.
- */
- protected static class Factory implements StateFactory {
- public static final redis.clients.jedis.JedisPoolConfig DEFAULT_POOL_CONFIG = new redis.clients.jedis.JedisPoolConfig();
-
- JedisClusterConfig jedisClusterConfig;
-
- StateType type;
- Serializer serializer;
- KeyFactory keyFactory;
- Options options;
-
- /**
- * Constructor
- *
- * @param jedisClusterConfig configuration for JedisCluster
- * @param type StateType
- * @param options options of State
- */
- public Factory(JedisClusterConfig jedisClusterConfig, StateType type, Options options) {
- this.jedisClusterConfig = jedisClusterConfig;
- this.type = type;
- this.options = options;
-
- this.keyFactory = options.keyFactory;
- if (this.keyFactory == null) {
- this.keyFactory = new KeyFactory.DefaultKeyFactory();
- }
- this.serializer = options.serializer;
- if (this.serializer == null) {
- this.serializer = DEFAULT_SERIALIZERS.get(type);
- if (this.serializer == null) {
- throw new RuntimeException("Couldn't find serializer for state type: " + type);
- }
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public State makeState(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
- JedisCluster jedisCluster = new JedisCluster(jedisClusterConfig.getNodes(),
- jedisClusterConfig.getTimeout(),
- jedisClusterConfig.getTimeout(),
- jedisClusterConfig.getMaxRedirections(),
- jedisClusterConfig.getPassword(),
- DEFAULT_POOL_CONFIG);
-
- RedisClusterMapState state = new RedisClusterMapState(jedisCluster, options, serializer, keyFactory);
- CachedMap c = new CachedMap(state, options.localCacheSize);
-
- MapState ms;
- if (type == StateType.NON_TRANSACTIONAL) {
- ms = NonTransactionalMap.build(c);
-
- } else if (type == StateType.OPAQUE) {
- ms = OpaqueMap.build(c);
-
- } else if (type == StateType.TRANSACTIONAL) {
- ms = TransactionalMap.build(c);
-
- } else {
- throw new RuntimeException("Unknown state type: " + type);
- }
-
- return new SnapshottableMap(ms, new Values(options.globalKey));
- }
- }
-
- private JedisCluster jedisCluster;
- private Options options;
- private Serializer serializer;
- private KeyFactory keyFactory;
-
- /**
- * Constructor
- *
- * @param jedisCluster JedisCluster
- * @param options options of State
- * @param serializer Serializer
- * @param keyFactory KeyFactory
- */
- public RedisClusterMapState(JedisCluster jedisCluster, Options options,
- Serializer<T> serializer, KeyFactory keyFactory) {
- this.jedisCluster = jedisCluster;
- this.options = options;
- this.serializer = serializer;
- this.keyFactory = keyFactory;
- }
-
- /**
* {@inheritDoc}
*/
@Override
@@ -350,4 +272,75 @@ public class RedisClusterMapState<T> extends AbstractRedisMapState<T> {
throw new IllegalArgumentException("Cannot process such data type: " + description.getDataType());
}
}
+
+ /**
+ * RedisClusterMapState.Factory provides Redis Cluster environment version of StateFactory.
+ */
+ protected static class Factory implements StateFactory {
+ public static final redis.clients.jedis.JedisPoolConfig DEFAULT_POOL_CONFIG = new redis.clients.jedis.JedisPoolConfig();
+
+ JedisClusterConfig jedisClusterConfig;
+
+ StateType type;
+ Serializer serializer;
+ KeyFactory keyFactory;
+ Options options;
+
+ /**
+ * Constructor
+ *
+ * @param jedisClusterConfig configuration for JedisCluster
+ * @param type StateType
+ * @param options options of State
+ */
+ public Factory(JedisClusterConfig jedisClusterConfig, StateType type, Options options) {
+ this.jedisClusterConfig = jedisClusterConfig;
+ this.type = type;
+ this.options = options;
+
+ this.keyFactory = options.keyFactory;
+ if (this.keyFactory == null) {
+ this.keyFactory = new KeyFactory.DefaultKeyFactory();
+ }
+ this.serializer = options.serializer;
+ if (this.serializer == null) {
+ this.serializer = DEFAULT_SERIALIZERS.get(type);
+ if (this.serializer == null) {
+ throw new RuntimeException("Couldn't find serializer for state type: " + type);
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public State makeState(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ JedisCluster jedisCluster = new JedisCluster(jedisClusterConfig.getNodes(),
+ jedisClusterConfig.getTimeout(),
+ jedisClusterConfig.getTimeout(),
+ jedisClusterConfig.getMaxRedirections(),
+ jedisClusterConfig.getPassword(),
+ DEFAULT_POOL_CONFIG);
+
+ RedisClusterMapState state = new RedisClusterMapState(jedisCluster, options, serializer, keyFactory);
+ CachedMap c = new CachedMap(state, options.localCacheSize);
+
+ MapState ms;
+ if (type == StateType.NON_TRANSACTIONAL) {
+ ms = NonTransactionalMap.build(c);
+
+ } else if (type == StateType.OPAQUE) {
+ ms = OpaqueMap.build(c);
+
+ } else if (type == StateType.TRANSACTIONAL) {
+ ms = TransactionalMap.build(c);
+
+ } else {
+ throw new RuntimeException("Unknown state type: " + type);
+ }
+
+ return new SnapshottableMap(ms, new Values(options.globalKey));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterState.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterState.java
index 11d8907..7bdd86b 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterState.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterState.java
@@ -1,37 +1,39 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.trident.state;
-import org.apache.storm.task.IMetricsContext;
+import java.util.Map;
import org.apache.storm.redis.common.config.JedisClusterConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import redis.clients.jedis.JedisCluster;
+import org.apache.storm.task.IMetricsContext;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.state.StateFactory;
-
-import java.util.Map;
+import redis.clients.jedis.JedisCluster;
/**
* Implementation of State for Redis Cluster environment.
*/
public class RedisClusterState implements State {
+ private JedisCluster jedisCluster;
+
+ /**
+ * Constructor
+ *
+ * @param jedisCluster JedisCluster
+ */
+ public RedisClusterState(JedisCluster jedisCluster) {
+ this.jedisCluster = jedisCluster;
+ }
+
/**
* {@inheritDoc}
*/
@@ -47,6 +49,26 @@ public class RedisClusterState implements State {
}
/**
+ * Borrows JedisCluster instance.
+ * <p/>
+ * Note that you should return borrowed instance when you finish using instance.
+ *
+ * @return JedisCluster instance
+ */
+ public JedisCluster getJedisCluster() {
+ return this.jedisCluster;
+ }
+
+ /**
+ * Returns JedisCluster instance to pool.
+ *
+ * @param jedisCluster JedisCluster instance to return to pool
+ */
+ public void returnJedisCluster(JedisCluster jedisCluster) {
+ //do nothing
+ }
+
+ /**
* RedisClusterState.Factory implements StateFactory for Redis Cluster environment.
*
* @see StateFactory
@@ -71,45 +93,14 @@ public class RedisClusterState implements State {
@Override
public State makeState(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
JedisCluster jedisCluster = new JedisCluster(jedisClusterConfig.getNodes(),
- jedisClusterConfig.getTimeout(),
- jedisClusterConfig.getTimeout(),
- jedisClusterConfig.getMaxRedirections(),
- jedisClusterConfig.getPassword(),
- DEFAULT_POOL_CONFIG);
+ jedisClusterConfig.getTimeout(),
+ jedisClusterConfig.getTimeout(),
+ jedisClusterConfig.getMaxRedirections(),
+ jedisClusterConfig.getPassword(),
+ DEFAULT_POOL_CONFIG);
return new RedisClusterState(jedisCluster);
}
}
- private JedisCluster jedisCluster;
-
- /**
- * Constructor
- *
- * @param jedisCluster JedisCluster
- */
- public RedisClusterState(JedisCluster jedisCluster) {
- this.jedisCluster = jedisCluster;
- }
-
- /**
- * Borrows JedisCluster instance.
- * <p/>
- * Note that you should return borrowed instance when you finish using instance.
- *
- * @return JedisCluster instance
- */
- public JedisCluster getJedisCluster() {
- return this.jedisCluster;
- }
-
- /**
- * Returns JedisCluster instance to pool.
- *
- * @param jedisCluster JedisCluster instance to return to pool
- */
- public void returnJedisCluster(JedisCluster jedisCluster) {
- //do nothing
- }
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateQuerier.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateQuerier.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateQuerier.java
index 6938409..b7401aa 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateQuerier.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateQuerier.java
@@ -1,28 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.trident.state;
-import org.apache.storm.redis.common.mapper.RedisLookupMapper;
-import redis.clients.jedis.JedisCluster;
-
import java.util.ArrayList;
import java.util.List;
+import org.apache.storm.redis.common.mapper.RedisLookupMapper;
+import redis.clients.jedis.JedisCluster;
/**
* BaseQueryFunction implementation for Redis Cluster environment.
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateUpdater.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateUpdater.java
index 1cc0725..0258ae5 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateUpdater.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisClusterStateUpdater.java
@@ -1,28 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.trident.state;
+import java.util.Map;
import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
import org.apache.storm.redis.common.mapper.RedisStoreMapper;
import redis.clients.jedis.JedisCluster;
-import java.util.Map;
-
/**
* BaseStateUpdater implementation for Redis Cluster environment.
*
@@ -63,25 +57,25 @@ public class RedisClusterStateUpdater extends AbstractRedisStateUpdater<RedisClu
String value = kvEntry.getValue();
switch (dataType) {
- case STRING:
- if (this.expireIntervalSec > 0) {
- jedisCluster.setex(key, expireIntervalSec, value);
- } else {
- jedisCluster.set(key, value);
- }
- break;
- case HASH:
- jedisCluster.hset(additionalKey, key, value);
- break;
- default:
- throw new IllegalArgumentException("Cannot process such data type: " + dataType);
+ case STRING:
+ if (this.expireIntervalSec > 0) {
+ jedisCluster.setex(key, expireIntervalSec, value);
+ } else {
+ jedisCluster.set(key, value);
+ }
+ break;
+ case HASH:
+ jedisCluster.hset(additionalKey, key, value);
+ break;
+ default:
+ throw new IllegalArgumentException("Cannot process such data type: " + dataType);
}
}
// send expire command for hash only once
// it expires key itself entirely, so use it with caution
if (dataType == RedisDataTypeDescription.RedisDataType.HASH &&
- this.expireIntervalSec > 0) {
+ this.expireIntervalSec > 0) {
jedisCluster.expire(additionalKey, expireIntervalSec);
}
} finally {
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisMapState.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisMapState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisMapState.java
index f68b5ef..6febc41 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisMapState.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisMapState.java
@@ -1,29 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.trident.state;
-import org.apache.storm.task.IMetricsContext;
-import org.apache.storm.tuple.Values;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.redis.common.config.JedisPoolConfig;
import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.JedisPool;
-import redis.clients.jedis.Pipeline;
+import org.apache.storm.task.IMetricsContext;
import org.apache.storm.trident.state.OpaqueValue;
import org.apache.storm.trident.state.Serializer;
import org.apache.storm.trident.state.State;
@@ -36,9 +29,10 @@ import org.apache.storm.trident.state.map.NonTransactionalMap;
import org.apache.storm.trident.state.map.OpaqueMap;
import org.apache.storm.trident.state.map.SnapshottableMap;
import org.apache.storm.trident.state.map.TransactionalMap;
-
-import java.util.List;
-import java.util.Map;
+import org.apache.storm.tuple.Values;
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
+import redis.clients.jedis.Pipeline;
/**
* IBackingMap implementation for single Redis environment.
@@ -47,6 +41,27 @@ import java.util.Map;
* @see AbstractRedisMapState
*/
public class RedisMapState<T> extends AbstractRedisMapState<T> {
+ private JedisPool jedisPool;
+ private Options options;
+ private Serializer serializer;
+ private KeyFactory keyFactory;
+
+ /**
+ * Constructor
+ *
+ * @param jedisPool JedisPool
+ * @param options options of State
+ * @param serializer Serializer
+ * @param keyFactory KeyFactory
+ */
+ public RedisMapState(JedisPool jedisPool, Options options,
+ Serializer<T> serializer, KeyFactory keyFactory) {
+ this.jedisPool = jedisPool;
+ this.options = options;
+ this.serializer = serializer;
+ this.keyFactory = keyFactory;
+ }
+
/**
* Provides StateFactory for opaque transactional.
*
@@ -189,97 +204,6 @@ public class RedisMapState<T> extends AbstractRedisMapState<T> {
}
/**
- * RedisMapState.Factory provides single Redis environment version of StateFactory.
- */
- protected static class Factory implements StateFactory {
- public static final redis.clients.jedis.JedisPoolConfig DEFAULT_POOL_CONFIG = new redis.clients.jedis.JedisPoolConfig();
-
- JedisPoolConfig jedisPoolConfig;
-
- StateType type;
- Serializer serializer;
- KeyFactory keyFactory;
- Options options;
-
- /**
- * Constructor
- *
- * @param jedisPoolConfig configuration for JedisPool
- * @param type StateType
- * @param options options of State
- */
- public Factory(JedisPoolConfig jedisPoolConfig, StateType type, Options options) {
- this.jedisPoolConfig = jedisPoolConfig;
- this.type = type;
- this.options = options;
-
- this.keyFactory = options.keyFactory;
- if (this.keyFactory == null) {
- this.keyFactory = new KeyFactory.DefaultKeyFactory();
- }
- this.serializer = options.serializer;
- if (this.serializer == null) {
- this.serializer = DEFAULT_SERIALIZERS.get(type);
- if (this.serializer == null) {
- throw new RuntimeException("Couldn't find serializer for state type: " + type);
- }
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public State makeState(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
- JedisPool jedisPool = new JedisPool(DEFAULT_POOL_CONFIG,
- jedisPoolConfig.getHost(),
- jedisPoolConfig.getPort(),
- jedisPoolConfig.getTimeout(),
- jedisPoolConfig.getPassword(),
- jedisPoolConfig.getDatabase());
- RedisMapState state = new RedisMapState(jedisPool, options, serializer, keyFactory);
- CachedMap c = new CachedMap(state, options.localCacheSize);
-
- MapState ms;
- if (type == StateType.NON_TRANSACTIONAL) {
- ms = NonTransactionalMap.build(c);
-
- } else if (type == StateType.OPAQUE) {
- ms = OpaqueMap.build(c);
-
- } else if (type == StateType.TRANSACTIONAL) {
- ms = TransactionalMap.build(c);
-
- } else {
- throw new RuntimeException("Unknown state type: " + type);
- }
-
- return new SnapshottableMap(ms, new Values(options.globalKey));
- }
- }
-
- private JedisPool jedisPool;
- private Options options;
- private Serializer serializer;
- private KeyFactory keyFactory;
-
- /**
- * Constructor
- *
- * @param jedisPool JedisPool
- * @param options options of State
- * @param serializer Serializer
- * @param keyFactory KeyFactory
- */
- public RedisMapState(JedisPool jedisPool, Options options,
- Serializer<T> serializer, KeyFactory keyFactory) {
- this.jedisPool = jedisPool;
- this.options = options;
- this.serializer = serializer;
- this.keyFactory = keyFactory;
- }
-
- /**
* {@inheritDoc}
*/
@Override
@@ -308,14 +232,14 @@ public class RedisMapState<T> extends AbstractRedisMapState<T> {
RedisDataTypeDescription description = this.options.dataTypeDescription;
switch (description.getDataType()) {
- case STRING:
- return jedis.mget(stringKeys);
+ case STRING:
+ return jedis.mget(stringKeys);
- case HASH:
- return jedis.hmget(description.getAdditionalKey(), stringKeys);
+ case HASH:
+ return jedis.hmget(description.getAdditionalKey(), stringKeys);
- default:
- throw new IllegalArgumentException("Cannot process such data type: " + description.getDataType());
+ default:
+ throw new IllegalArgumentException("Cannot process such data type: " + description.getDataType());
}
} finally {
@@ -337,27 +261,27 @@ public class RedisMapState<T> extends AbstractRedisMapState<T> {
RedisDataTypeDescription description = this.options.dataTypeDescription;
switch (description.getDataType()) {
- case STRING:
- String[] keyValue = buildKeyValuesList(keyValues);
- jedis.mset(keyValue);
- if(this.options.expireIntervalSec > 0){
- Pipeline pipe = jedis.pipelined();
- for(int i = 0; i < keyValue.length; i += 2){
- pipe.expire(keyValue[i], this.options.expireIntervalSec);
+ case STRING:
+ String[] keyValue = buildKeyValuesList(keyValues);
+ jedis.mset(keyValue);
+ if (this.options.expireIntervalSec > 0) {
+ Pipeline pipe = jedis.pipelined();
+ for (int i = 0; i < keyValue.length; i += 2) {
+ pipe.expire(keyValue[i], this.options.expireIntervalSec);
+ }
+ pipe.sync();
}
- pipe.sync();
- }
- break;
+ break;
- case HASH:
- jedis.hmset(description.getAdditionalKey(), keyValues);
- if (this.options.expireIntervalSec > 0) {
- jedis.expire(description.getAdditionalKey(), this.options.expireIntervalSec);
- }
- break;
+ case HASH:
+ jedis.hmset(description.getAdditionalKey(), keyValues);
+ if (this.options.expireIntervalSec > 0) {
+ jedis.expire(description.getAdditionalKey(), this.options.expireIntervalSec);
+ }
+ break;
- default:
- throw new IllegalArgumentException("Cannot process such data type: " + description.getDataType());
+ default:
+ throw new IllegalArgumentException("Cannot process such data type: " + description.getDataType());
}
} finally {
@@ -378,4 +302,74 @@ public class RedisMapState<T> extends AbstractRedisMapState<T> {
return keyValueLists;
}
+
+ /**
+ * RedisMapState.Factory provides single Redis environment version of StateFactory.
+ */
+ protected static class Factory implements StateFactory {
+ public static final redis.clients.jedis.JedisPoolConfig DEFAULT_POOL_CONFIG = new redis.clients.jedis.JedisPoolConfig();
+
+ JedisPoolConfig jedisPoolConfig;
+
+ StateType type;
+ Serializer serializer;
+ KeyFactory keyFactory;
+ Options options;
+
+ /**
+ * Constructor
+ *
+ * @param jedisPoolConfig configuration for JedisPool
+ * @param type StateType
+ * @param options options of State
+ */
+ public Factory(JedisPoolConfig jedisPoolConfig, StateType type, Options options) {
+ this.jedisPoolConfig = jedisPoolConfig;
+ this.type = type;
+ this.options = options;
+
+ this.keyFactory = options.keyFactory;
+ if (this.keyFactory == null) {
+ this.keyFactory = new KeyFactory.DefaultKeyFactory();
+ }
+ this.serializer = options.serializer;
+ if (this.serializer == null) {
+ this.serializer = DEFAULT_SERIALIZERS.get(type);
+ if (this.serializer == null) {
+ throw new RuntimeException("Couldn't find serializer for state type: " + type);
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public State makeState(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ JedisPool jedisPool = new JedisPool(DEFAULT_POOL_CONFIG,
+ jedisPoolConfig.getHost(),
+ jedisPoolConfig.getPort(),
+ jedisPoolConfig.getTimeout(),
+ jedisPoolConfig.getPassword(),
+ jedisPoolConfig.getDatabase());
+ RedisMapState state = new RedisMapState(jedisPool, options, serializer, keyFactory);
+ CachedMap c = new CachedMap(state, options.localCacheSize);
+
+ MapState ms;
+ if (type == StateType.NON_TRANSACTIONAL) {
+ ms = NonTransactionalMap.build(c);
+
+ } else if (type == StateType.OPAQUE) {
+ ms = OpaqueMap.build(c);
+
+ } else if (type == StateType.TRANSACTIONAL) {
+ ms = TransactionalMap.build(c);
+
+ } else {
+ throw new RuntimeException("Unknown state type: " + type);
+ }
+
+ return new SnapshottableMap(ms, new Values(options.globalKey));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisState.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisState.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisState.java
index ec0b679..76b60f7 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisState.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisState.java
@@ -1,37 +1,40 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.trident.state;
-import org.apache.storm.task.IMetricsContext;
+import java.util.Map;
import org.apache.storm.redis.common.config.JedisPoolConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.JedisPool;
+import org.apache.storm.task.IMetricsContext;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.state.StateFactory;
-
-import java.util.Map;
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
/**
* Implementation of State for single Redis environment.
*/
public class RedisState implements State {
+ private JedisPool jedisPool;
+
+ /**
+ * Constructor
+ *
+ * @param jedisPool JedisPool
+ */
+ public RedisState(JedisPool jedisPool) {
+ this.jedisPool = jedisPool;
+ }
+
/**
* {@inheritDoc}
*/
@@ -47,6 +50,26 @@ public class RedisState implements State {
}
/**
+ * Borrows Jedis instance from pool.
+ * <p/>
+ * Note that you should return borrowed instance to pool when you finish using instance.
+ *
+ * @return Jedis instance
+ */
+ public Jedis getJedis() {
+ return this.jedisPool.getResource();
+ }
+
+ /**
+ * Returns Jedis instance to pool.
+ *
+ * @param jedis Jedis instance to return to pool
+ */
+ public void returnJedis(Jedis jedis) {
+ jedis.close();
+ }
+
+ /**
* RedisState.Factory implements StateFactory for single Redis environment.
*
* @see StateFactory
@@ -81,35 +104,4 @@ public class RedisState implements State {
}
}
- private JedisPool jedisPool;
-
- /**
- * Constructor
- *
- * @param jedisPool JedisPool
- */
- public RedisState(JedisPool jedisPool) {
- this.jedisPool = jedisPool;
- }
-
- /**
- * Borrows Jedis instance from pool.
- * <p/>
- * Note that you should return borrowed instance to pool when you finish using instance.
- *
- * @return Jedis instance
- */
- public Jedis getJedis() {
- return this.jedisPool.getResource();
- }
-
- /**
- * Returns Jedis instance to pool.
- *
- * @param jedis Jedis instance to return to pool
- */
- public void returnJedis(Jedis jedis) {
- jedis.close();
- }
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateQuerier.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateQuerier.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateQuerier.java
index d76fec2..8be00e2 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateQuerier.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateQuerier.java
@@ -1,27 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.trident.state;
+import java.util.List;
import org.apache.storm.redis.common.mapper.RedisLookupMapper;
import redis.clients.jedis.Jedis;
-import java.util.List;
-
/**
* BaseQueryFunction implementation for single Redis environment.
*
@@ -49,14 +43,14 @@ public class RedisStateQuerier extends AbstractRedisStateQuerier<RedisState> {
String[] keysForRedis = keys.toArray(new String[keys.size()]);
switch (dataType) {
- case STRING:
- redisVals = jedis.mget(keysForRedis);
- break;
- case HASH:
- redisVals = jedis.hmget(additionalKey, keysForRedis);
- break;
- default:
- throw new IllegalArgumentException("Cannot process such data type: " + dataType);
+ case STRING:
+ redisVals = jedis.mget(keysForRedis);
+ break;
+ case HASH:
+ redisVals = jedis.hmget(additionalKey, keysForRedis);
+ break;
+ default:
+ throw new IllegalArgumentException("Cannot process such data type: " + dataType);
}
return redisVals;
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateUpdater.java b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateUpdater.java
index d0507cf..0a5aede 100644
--- a/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateUpdater.java
+++ b/external/storm-redis/src/main/java/org/apache/storm/redis/trident/state/RedisStateUpdater.java
@@ -1,29 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.trident.state;
+import java.util.Map;
import org.apache.storm.redis.common.mapper.RedisDataTypeDescription;
import org.apache.storm.redis.common.mapper.RedisStoreMapper;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.Pipeline;
-import java.util.Map;
-
/**
* BaseStateUpdater implementation for single Redis environment.
*
@@ -65,25 +59,25 @@ public class RedisStateUpdater extends AbstractRedisStateUpdater<RedisState> {
String value = kvEntry.getValue();
switch (dataType) {
- case STRING:
- if (this.expireIntervalSec > 0) {
- pipeline.setex(key, expireIntervalSec, value);
- } else {
- pipeline.set(key, value);
- }
- break;
- case HASH:
- pipeline.hset(additionalKey, key, value);
- break;
- default:
- throw new IllegalArgumentException("Cannot process such data type: " + dataType);
+ case STRING:
+ if (this.expireIntervalSec > 0) {
+ pipeline.setex(key, expireIntervalSec, value);
+ } else {
+ pipeline.set(key, value);
+ }
+ break;
+ case HASH:
+ pipeline.hset(additionalKey, key, value);
+ break;
+ default:
+ throw new IllegalArgumentException("Cannot process such data type: " + dataType);
}
}
// send expire command for hash only once
// it expires key itself entirely, so use it with caution
if (dataType == RedisDataTypeDescription.RedisDataType.HASH &&
- this.expireIntervalSec > 0) {
+ this.expireIntervalSec > 0) {
pipeline.expire(additionalKey, expireIntervalSec);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateIteratorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateIteratorTest.java b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateIteratorTest.java
index 565a250..fb0052d 100644
--- a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateIteratorTest.java
+++ b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateIteratorTest.java
@@ -1,24 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.redis.state;
import com.google.common.primitives.UnsignedBytes;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
import org.apache.storm.redis.common.adapter.RedisCommandsAdapterJedis;
import org.apache.storm.redis.common.container.RedisCommandsInstanceContainer;
import org.apache.storm.state.DefaultStateEncoder;
@@ -29,11 +27,6 @@ import org.junit.Test;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -79,15 +72,15 @@ public class RedisKeyValueStateIteratorTest {
putEncodedKeyValueToMap(chunkMap, "key2".getBytes(), "value2".getBytes());
ScanResult<Map.Entry<byte[], byte[]>> scanResultFirst = new ScanResult<>(
- "12345".getBytes(), new ArrayList<>(chunkMap.entrySet()));
+ "12345".getBytes(), new ArrayList<>(chunkMap.entrySet()));
ScanResult<Map.Entry<byte[], byte[]>> scanResultSecond = new ScanResult<>(
- ScanParams.SCAN_POINTER_START_BINARY, new ArrayList<Map.Entry<byte[], byte[]>>());
+ ScanParams.SCAN_POINTER_START_BINARY, new ArrayList<Map.Entry<byte[], byte[]>>());
when(mockJedis.hscan(eq(namespace), any(byte[].class), any(ScanParams.class)))
- .thenReturn(scanResultFirst, scanResultSecond);
+ .thenReturn(scanResultFirst, scanResultSecond);
RedisKeyValueStateIterator<byte[], byte[]> kvIterator =
- new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(),
- pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
+ new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(),
+ pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
assertNextEntry(kvIterator, "key0".getBytes(), "value0".getBytes());
@@ -112,17 +105,17 @@ public class RedisKeyValueStateIteratorTest {
putEncodedKeyValueToMap(chunkMap, "key2".getBytes(), "value2".getBytes());
ScanResult<Map.Entry<byte[], byte[]>> scanResultFirst = new ScanResult<>(
- "12345".getBytes(), new ArrayList<Map.Entry<byte[], byte[]>>());
+ "12345".getBytes(), new ArrayList<Map.Entry<byte[], byte[]>>());
ScanResult<Map.Entry<byte[], byte[]>> scanResultSecond = new ScanResult<>(
- "23456".getBytes(), new ArrayList<Map.Entry<byte[], byte[]>>());
+ "23456".getBytes(), new ArrayList<Map.Entry<byte[], byte[]>>());
ScanResult<Map.Entry<byte[], byte[]>> scanResultThird = new ScanResult<>(
- ScanParams.SCAN_POINTER_START_BINARY, new ArrayList<>(chunkMap.entrySet()));
+ ScanParams.SCAN_POINTER_START_BINARY, new ArrayList<>(chunkMap.entrySet()));
when(mockJedis.hscan(eq(namespace), any(byte[].class), any(ScanParams.class)))
- .thenReturn(scanResultFirst, scanResultSecond, scanResultThird);
+ .thenReturn(scanResultFirst, scanResultSecond, scanResultThird);
RedisKeyValueStateIterator<byte[], byte[]> kvIterator =
- new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(),
- pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
+ new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(),
+ pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
assertNextEntry(kvIterator, "key0".getBytes(), "value0".getBytes());
@@ -152,17 +145,17 @@ public class RedisKeyValueStateIteratorTest {
putEncodedKeyValueToMap(chunkMap2, "key4".getBytes(), "value4".getBytes());
ScanResult<Map.Entry<byte[], byte[]>> scanResultFirst = new ScanResult<>(
- "12345".getBytes(), new ArrayList<>(chunkMap.entrySet()));
+ "12345".getBytes(), new ArrayList<>(chunkMap.entrySet()));
ScanResult<Map.Entry<byte[], byte[]>> scanResultSecond = new ScanResult<>(
- "23456".getBytes(), new ArrayList<>(chunkMap2.entrySet()));
+ "23456".getBytes(), new ArrayList<>(chunkMap2.entrySet()));
ScanResult<Map.Entry<byte[], byte[]>> scanResultThird = new ScanResult<>(
- ScanParams.SCAN_POINTER_START_BINARY, new ArrayList<Map.Entry<byte[], byte[]>>());
+ ScanParams.SCAN_POINTER_START_BINARY, new ArrayList<Map.Entry<byte[], byte[]>>());
when(mockJedis.hscan(eq(namespace), any(byte[].class), any(ScanParams.class)))
- .thenReturn(scanResultFirst, scanResultSecond, scanResultThird);
+ .thenReturn(scanResultFirst, scanResultSecond, scanResultThird);
RedisKeyValueStateIterator<byte[], byte[]> kvIterator =
- new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(),
- pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
+ new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(),
+ pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
// keys shouldn't appear twice
@@ -184,13 +177,13 @@ public class RedisKeyValueStateIteratorTest {
NavigableMap<byte[], byte[]> pendingCommit = getBinaryTreeMap();
ScanResult<Map.Entry<byte[], byte[]>> scanResult = new ScanResult<>(
- ScanParams.SCAN_POINTER_START_BINARY, new ArrayList<Map.Entry<byte[], byte[]>>());
+ ScanParams.SCAN_POINTER_START_BINARY, new ArrayList<Map.Entry<byte[], byte[]>>());
when(mockJedis.hscan(eq(namespace), any(byte[].class), any(ScanParams.class)))
- .thenReturn(scanResult);
+ .thenReturn(scanResult);
RedisKeyValueStateIterator<byte[], byte[]> kvIterator =
- new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(),
- pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
+ new RedisKeyValueStateIterator<>(namespace, mockContainer, pendingPrepare.entrySet().iterator(),
+ pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
assertFalse(kvIterator.hasNext());
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateProviderTest.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateProviderTest.java b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateProviderTest.java
index 92e76ec..b8de3d3 100644
--- a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateProviderTest.java
+++ b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateProviderTest.java
@@ -1,32 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.redis.state;
-import org.apache.storm.Config;
-import org.apache.storm.state.State;
-import org.junit.Assert;
-import org.junit.Test;
+package org.apache.storm.redis.state;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import org.apache.storm.Config;
+import org.junit.Test;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
/**
* Unit tests for {@link RedisKeyValueStateProvider}
@@ -49,8 +41,8 @@ public class RedisKeyValueStateProviderTest {
RedisKeyValueStateProvider provider = new RedisKeyValueStateProvider();
Map<String, Object> topoConf = new HashMap<>();
topoConf.put(Config.TOPOLOGY_STATE_PROVIDER_CONFIG, "{\"keyClass\":\"String\", \"valueClass\":\"String\"," +
- " \"jedisPoolConfig\":" +
- "{\"host\":\"localhost\", \"port\":1000}}");
+ " \"jedisPoolConfig\":" +
+ "{\"host\":\"localhost\", \"port\":1000}}");
RedisKeyValueStateProvider.StateConfig config = provider.getStateConfig(topoConf);
//System.out.println(config);
http://git-wip-us.apache.org/repos/asf/storm/blob/e3f5b138/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateTest.java
----------------------------------------------------------------------
diff --git a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateTest.java b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateTest.java
index 480001c..db98a39 100644
--- a/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateTest.java
+++ b/external/storm-redis/src/test/java/org/apache/storm/redis/state/RedisKeyValueStateTest.java
@@ -1,23 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.redis.state;
import com.google.common.primitives.UnsignedBytes;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.storm.redis.common.commands.RedisCommands;
import org.apache.storm.redis.common.container.RedisCommandsInstanceContainer;
import org.apache.storm.state.DefaultStateSerializer;
@@ -29,14 +30,8 @@ import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import redis.clients.util.SafeEncoder;
-import java.util.HashMap;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
/**
* Unit tests for {@link RedisKeyValueState}
@@ -53,7 +48,7 @@ public class RedisKeyValueStateTest {
@Before
public void setUp() {
final NavigableMap<byte[], NavigableMap<byte[], byte[]>> mockMap =
- new ConcurrentSkipListMap<>(UnsignedBytes.lexicographicalComparator());
+ new ConcurrentSkipListMap<>(UnsignedBytes.lexicographicalComparator());
mockContainer = Mockito.mock(RedisCommandsInstanceContainer.class);
mockCommands = Mockito.mock(RedisCommands.class);
Mockito.when(mockContainer.getInstance()).thenReturn(mockCommands);
@@ -62,78 +57,78 @@ public class RedisKeyValueStateTest {
ArgumentCaptor<Map> mapArgumentCaptor = ArgumentCaptor.forClass(Map.class);
Mockito.when(mockCommands.exists(Mockito.any(byte[].class)))
- .thenAnswer(new Answer<Boolean>() {
- @Override
- public Boolean answer(InvocationOnMock invocation) throws Throwable {
- Object[] args = invocation.getArguments();
- return exists(mockMap, (byte[]) args[0]);
- }
- });
+ .thenAnswer(new Answer<Boolean>() {
+ @Override
+ public Boolean answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ return exists(mockMap, (byte[]) args[0]);
+ }
+ });
Mockito.when(mockCommands.del(Mockito.any(byte[].class)))
- .thenAnswer(new Answer<Long>() {
- @Override
- public Long answer(InvocationOnMock invocation) throws Throwable {
- Object[] args = invocation.getArguments();
- return del(mockMap, (byte[]) args[0]);
- }
- });
+ .thenAnswer(new Answer<Long>() {
+ @Override
+ public Long answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ return del(mockMap, (byte[]) args[0]);
+ }
+ });
Mockito.when(mockCommands.hmset(Mockito.any(byte[].class), Mockito.anyMap()))
- .thenAnswer(new Answer<String>() {
- @Override
- public String answer(InvocationOnMock invocation) throws Throwable {
- Object[] args = invocation.getArguments();
- return hmset(mockMap, (byte[]) args[0], (Map<byte[], byte[]>) args[1]);
- }
- });
+ .thenAnswer(new Answer<String>() {
+ @Override
+ public String answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ return hmset(mockMap, (byte[]) args[0], (Map<byte[], byte[]>) args[1]);
+ }
+ });
Mockito.when(mockCommands.hget(Mockito.any(byte[].class), Mockito.any(byte[].class)))
- .thenAnswer(new Answer<byte[]>() {
- @Override
- public byte[] answer(InvocationOnMock invocation) throws Throwable {
- Object[] args = invocation.getArguments();
- return hget(mockMap, (byte[]) args[0], (byte[]) args[1]);
- }
- });
+ .thenAnswer(new Answer<byte[]>() {
+ @Override
+ public byte[] answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ return hget(mockMap, (byte[]) args[0], (byte[]) args[1]);
+ }
+ });
Mockito.when(mockCommands.hdel(Mockito.any(byte[].class), Mockito.<byte[]>anyVararg()))
- .thenAnswer(new Answer<Long>() {
- @Override
- public Long answer(InvocationOnMock invocation) throws Throwable {
- Object[] args = invocation.getArguments();
- int argsSize = args.length;
- byte[][] fields = Arrays.asList(args).subList(1, argsSize).toArray(new byte[argsSize - 1][]);
- return hdel(mockMap, (byte[]) args[0], fields);
- }
- });
+ .thenAnswer(new Answer<Long>() {
+ @Override
+ public Long answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ int argsSize = args.length;
+ byte[][] fields = Arrays.asList(args).subList(1, argsSize).toArray(new byte[argsSize - 1][]);
+ return hdel(mockMap, (byte[]) args[0], fields);
+ }
+ });
Mockito.when(mockCommands.exists(Mockito.anyString()))
- .thenAnswer(new Answer<Boolean>() {
- @Override
- public Boolean answer(InvocationOnMock invocation) throws Throwable {
- Object[] args = invocation.getArguments();
- return exists(mockMap, (String) args[0]);
- }
- });
+ .thenAnswer(new Answer<Boolean>() {
+ @Override
+ public Boolean answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ return exists(mockMap, (String) args[0]);
+ }
+ });
Mockito.when(mockCommands.hmset(Mockito.anyString(), Mockito.anyMap()))
- .thenAnswer(new Answer<String>() {
- @Override
- public String answer(InvocationOnMock invocation) throws Throwable {
- Object[] args = invocation.getArguments();
- return hmset(mockMap, (String) args[0], (Map<String, String>) args[1]);
- }
- });
+ .thenAnswer(new Answer<String>() {
+ @Override
+ public String answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ return hmset(mockMap, (String) args[0], (Map<String, String>) args[1]);
+ }
+ });
Mockito.when(mockCommands.hgetAll(Mockito.anyString()))
- .thenAnswer(new Answer<Map<String, String>>() {
- @Override
- public Map<String, String> answer(InvocationOnMock invocation) throws Throwable {
- Object[] args = invocation.getArguments();
- return hgetAll(mockMap, (String) args[0]);
- }
- });
+ .thenAnswer(new Answer<Map<String, String>>() {
+ @Override
+ public Map<String, String> answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ return hgetAll(mockMap, (String) args[0]);
+ }
+ });
keyValueState = new RedisKeyValueState<String, String>("test", mockContainer, new DefaultStateSerializer<String>(),
new DefaultStateSerializer<String>());
@@ -167,38 +162,38 @@ public class RedisKeyValueStateTest {
keyValueState.put("b", "2");
keyValueState.prepareCommit(1);
keyValueState.put("c", "3");
- assertArrayEquals(new String[]{"1", "2", "3"}, getValues());
+ assertArrayEquals(new String[]{ "1", "2", "3" }, getValues());
keyValueState.rollback();
- assertArrayEquals(new String[]{null, null, null}, getValues());
+ assertArrayEquals(new String[]{ null, null, null }, getValues());
keyValueState.put("a", "1");
keyValueState.put("b", "2");
keyValueState.prepareCommit(1);
keyValueState.commit(1);
keyValueState.put("c", "3");
- assertArrayEquals(new String[]{"1", "2", "3"}, getValues());
+ assertArrayEquals(new String[]{ "1", "2", "3" }, getValues());
keyValueState.rollback();
- assertArrayEquals(new String[]{"1", "2", null}, getValues());
+ assertArrayEquals(new String[]{ "1", "2", null }, getValues());
keyValueState.put("c", "3");
assertEquals("2", keyValueState.delete("b"));
assertEquals("3", keyValueState.delete("c"));
- assertArrayEquals(new String[]{"1", null, null}, getValues());
+ assertArrayEquals(new String[]{ "1", null, null }, getValues());
keyValueState.prepareCommit(2);
- assertArrayEquals(new String[]{"1", null, null}, getValues());
+ assertArrayEquals(new String[]{ "1", null, null }, getValues());
keyValueState.commit(2);
- assertArrayEquals(new String[]{"1", null, null}, getValues());
+ assertArrayEquals(new String[]{ "1", null, null }, getValues());
keyValueState.put("b", "2");
keyValueState.prepareCommit(3);
keyValueState.put("c", "3");
- assertArrayEquals(new String[]{"1", "2", "3"}, getValues());
+ assertArrayEquals(new String[]{ "1", "2", "3" }, getValues());
keyValueState.rollback();
- assertArrayEquals(new String[]{"1", null, null}, getValues());
+ assertArrayEquals(new String[]{ "1", null, null }, getValues());
}
private String[] getValues() {
return new String[]{
- keyValueState.get("a"),
- keyValueState.get("b"),
- keyValueState.get("c")
+ keyValueState.get("a"),
+ keyValueState.get("b"),
+ keyValueState.get("c")
};
}
@@ -221,10 +216,11 @@ public class RedisKeyValueStateTest {
}
private Long del(NavigableMap<byte[], NavigableMap<byte[], byte[]>> mockMap, byte[] key) {
- if (mockMap.remove(key) == null)
+ if (mockMap.remove(key) == null) {
return 0L;
- else
+ } else {
return 1L;
+ }
}
private byte[] hget(NavigableMap<byte[], NavigableMap<byte[], byte[]>> mockMap, byte[] namespace, byte[] key) {
@@ -234,9 +230,9 @@ public class RedisKeyValueStateTest {
return null;
}
- private Long hdel(NavigableMap<byte[], NavigableMap<byte[], byte[]>> mockMap, byte[] namespace, byte[] ... keys) {
+ private Long hdel(NavigableMap<byte[], NavigableMap<byte[], byte[]>> mockMap, byte[] namespace, byte[]... keys) {
Long count = 0L;
- for (byte[] key: keys) {
+ for (byte[] key : keys) {
if (mockMap.get(namespace).remove(key) != null) count++;
}
return count;
[14/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Slot.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Slot.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Slot.java
index 5a629ad..1dda41c 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Slot.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Slot.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.daemon.supervisor;
@@ -72,212 +66,85 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
StormMetricsRegistry.registerMeter("supervisor:num-workers-killed-hb-null");
private static final Meter numForceKill =
StormMetricsRegistry.registerMeter("supervisor:num-workers-force-kill");
+ private static final long ONE_SEC_IN_NANO = TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS);
+ private final AtomicReference<LocalAssignment> newAssignment = new AtomicReference<>();
+ private final AtomicReference<Set<TopoProfileAction>> profiling = new AtomicReference<>(new HashSet<>());
- static enum MachineState {
- EMPTY,
- RUNNING,
- WAITING_FOR_WORKER_START,
- KILL_AND_RELAUNCH,
- KILL,
- KILL_BLOB_UPDATE,
- WAITING_FOR_BLOB_LOCALIZATION,
- WAITING_FOR_BLOB_UPDATE;
- }
-
- static class StaticState {
- public final AsyncLocalizer localizer;
- public final long hbTimeoutMs;
- public final long firstHbTimeoutMs;
- public final long killSleepMs;
- public final long monitorFreqMs;
- public final ContainerLauncher containerLauncher;
- public final int port;
- public final String host;
- public final ISupervisor iSupervisor;
- public final LocalState localState;
- public final BlobChangingCallback changingCallback;
- public final OnlyLatestExecutor<Integer> metricsExec;
- public final WorkerMetricsProcessor metricsProcessor;
-
- StaticState(AsyncLocalizer localizer, long hbTimeoutMs, long firstHbTimeoutMs,
- long killSleepMs, long monitorFreqMs,
- ContainerLauncher containerLauncher, String host, int port,
- ISupervisor iSupervisor, LocalState localState,
- BlobChangingCallback changingCallback,
- OnlyLatestExecutor<Integer> metricsExec,
- WorkerMetricsProcessor metricsProcessor) {
- this.localizer = localizer;
- this.hbTimeoutMs = hbTimeoutMs;
- this.firstHbTimeoutMs = firstHbTimeoutMs;
- this.containerLauncher = containerLauncher;
- this.killSleepMs = killSleepMs;
- this.monitorFreqMs = monitorFreqMs;
- this.host = host;
- this.port = port;
- this.iSupervisor = iSupervisor;
- this.localState = localState;
- this.changingCallback = changingCallback;
- this.metricsExec = metricsExec;
- this.metricsProcessor = metricsProcessor;
- }
- }
+ ;
+ private final BlockingQueue<BlobChanging> changingBlobs = new LinkedBlockingQueue<>();
+ private final StaticState staticState;
+ private final IStormClusterState clusterState;
+ private final AtomicReference<Map<Long, LocalAssignment>> cachedCurrentAssignments;
+ private final OnlyLatestExecutor<Integer> metricsExec;
+ private volatile boolean done = false;
+ private volatile DynamicState dynamicState;
- static class DynamicState {
- public final MachineState state;
- public final LocalAssignment newAssignment;
- public final LocalAssignment currentAssignment;
- public final Container container;
- public final LocalAssignment pendingLocalization;
- public final Future<Void> pendingDownload;
- public final Set<TopoProfileAction> profileActions;
- public final Set<TopoProfileAction> pendingStopProfileActions;
- public final Set<BlobChanging> changingBlobs;
- public final LocalAssignment pendingChangingBlobsAssignment;
- public final Set<Future<Void>> pendingChangingBlobs;
-
- /**
- * The last time that WAITING_FOR_WORKER_START, KILL, or KILL_AND_RELAUNCH were entered into.
- */
- public final long startTime;
-
- public DynamicState(final LocalAssignment currentAssignment, Container container, final LocalAssignment newAssignment) {
- this.currentAssignment = currentAssignment;
- this.container = container;
- if ((currentAssignment == null) ^ (container == null)) {
- throw new IllegalArgumentException("Container and current assignment must both be null, or neither can be null");
- }
-
- if (currentAssignment == null) {
- state = MachineState.EMPTY;
- } else {
- state = MachineState.RUNNING;
- }
-
- this.startTime = System.currentTimeMillis();
- this.newAssignment = newAssignment;
- this.pendingLocalization = null;
- this.pendingDownload = null;
- this.profileActions = Collections.emptySet();
- this.pendingStopProfileActions = Collections.emptySet();
- this.changingBlobs = Collections.emptySet();
- this.pendingChangingBlobsAssignment = null;
- this.pendingChangingBlobs = Collections.emptySet();
- }
+ public Slot(AsyncLocalizer localizer, Map<String, Object> conf,
+ ContainerLauncher containerLauncher, String host,
+ int port, LocalState localState,
+ IStormClusterState clusterState,
+ ISupervisor iSupervisor,
+ AtomicReference<Map<Long, LocalAssignment>> cachedCurrentAssignments,
+ OnlyLatestExecutor<Integer> metricsExec,
+ WorkerMetricsProcessor metricsProcessor) throws Exception {
+ super("SLOT_" + port);
+ this.metricsExec = metricsExec;
- public DynamicState(final MachineState state, final LocalAssignment newAssignment,
- final Container container, final LocalAssignment currentAssignment,
- final LocalAssignment pendingLocalization, final long startTime,
- final Future<Void> pendingDownload, final Set<TopoProfileAction> profileActions,
- final Set<TopoProfileAction> pendingStopProfileActions,
- final Set<BlobChanging> changingBlobs,
- final Set<Future<Void>> pendingChangingBlobs, final LocalAssignment pendingChaningBlobsAssignment) {
- assert pendingChangingBlobs != null;
- assert !(pendingChangingBlobs.isEmpty() ^ (pendingChaningBlobsAssignment == null));
- this.state = state;
- this.newAssignment = newAssignment;
- this.currentAssignment = currentAssignment;
- this.container = container;
- this.pendingLocalization = pendingLocalization;
- this.startTime = startTime;
- this.pendingDownload = pendingDownload;
- this.profileActions = profileActions;
- this.pendingStopProfileActions = pendingStopProfileActions;
- this.changingBlobs = changingBlobs;
- this.pendingChangingBlobs = pendingChangingBlobs;
- this.pendingChangingBlobsAssignment = pendingChaningBlobsAssignment;
+ this.cachedCurrentAssignments = cachedCurrentAssignments;
+ this.clusterState = clusterState;
+ Map<Integer, LocalAssignment> assignments = localState.getLocalAssignmentsMap();
+ LocalAssignment currentAssignment = null;
+ if (assignments != null) {
+ currentAssignment = assignments.get(port);
}
-
- public String toString() {
- StringBuffer sb = new StringBuffer();
- sb.append(state);
- sb.append(" msInState: ");
- sb.append(Time.currentTimeMillis() - startTime);
- if (container != null) {
- sb.append(" ");
- sb.append(container);
+ Container container = null;
+ if (currentAssignment != null) {
+ try {
+ // For now we do not make a transaction when removing a topology assignment from local, an overdue
+ // assignment may be left on local disk.
+ // So we should check if the local disk assignment is valid when initializing:
+ // if topology files does not exist, the worker[possibly alive] will be reassigned if it is timed-out;
+ // if topology files exist but the topology id is invalid, just let Supervisor make a sync;
+ // if topology files exist and topology files is valid, recover the container.
+ if (ClientSupervisorUtils.doRequiredTopoFilesExist(conf, currentAssignment.get_topology_id())) {
+ container = containerLauncher.recoverContainer(port, currentAssignment, localState);
+ } else {
+ // Make the assignment null to let slot clean up the disk assignment.
+ currentAssignment = null;
+ }
+ } catch (ContainerRecoveryException e) {
+ //We could not recover container will be null.
}
- return sb.toString();
- }
-
- /**
- * Set the new assignment for the state. This should never be called from within the state machine.
- * It is an input from outside.
- * @param newAssignment the new assignment to set
- * @return the updated DynamicState.
- */
- public DynamicState withNewAssignment(LocalAssignment newAssignment) {
- return new DynamicState(this.state, newAssignment,
- this.container, this.currentAssignment,
- this.pendingLocalization, this.startTime,
- this.pendingDownload, this.profileActions,
- this.pendingStopProfileActions, this.changingBlobs,
- this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
- }
-
- public DynamicState withPendingLocalization(LocalAssignment pendingLocalization, Future<Void> pendingDownload) {
- return new DynamicState(this.state, this.newAssignment,
- this.container, this.currentAssignment,
- pendingLocalization, this.startTime,
- pendingDownload, this.profileActions,
- this.pendingStopProfileActions, this.changingBlobs,
- this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
- }
-
- public DynamicState withPendingLocalization(Future<Void> pendingDownload) {
- return withPendingLocalization(this.pendingLocalization, pendingDownload);
- }
-
- public DynamicState withState(final MachineState state) {
- long newStartTime = Time.currentTimeMillis();
- return new DynamicState(state, this.newAssignment,
- this.container, this.currentAssignment,
- this.pendingLocalization, newStartTime,
- this.pendingDownload, this.profileActions,
- this.pendingStopProfileActions, this.changingBlobs,
- this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
- }
-
- public DynamicState withCurrentAssignment(final Container container, final LocalAssignment currentAssignment) {
- return new DynamicState(this.state, this.newAssignment,
- container, currentAssignment,
- this.pendingLocalization, this.startTime,
- this.pendingDownload, this.profileActions,
- this.pendingStopProfileActions, this.changingBlobs,
- this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
- }
-
- public DynamicState withProfileActions(Set<TopoProfileAction> profileActions, Set<TopoProfileAction> pendingStopProfileActions) {
- return new DynamicState(this.state, this.newAssignment,
- this.container, this.currentAssignment,
- this.pendingLocalization, this.startTime,
- this.pendingDownload, profileActions,
- pendingStopProfileActions, this.changingBlobs,
- this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
}
- public DynamicState withChangingBlobs(Set<BlobChanging> changingBlobs) {
- if (changingBlobs == this.changingBlobs) {
- return this;
- }
- return new DynamicState(this.state, this.newAssignment,
- this.container, this.currentAssignment,
- this.pendingLocalization, this.startTime,
- this.pendingDownload, profileActions,
- this.pendingStopProfileActions, changingBlobs,
- this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
+ LocalAssignment newAssignment = currentAssignment;
+ if (currentAssignment != null && container == null) {
+ currentAssignment = null;
+ //Assigned something but it is not running
}
- public DynamicState withPendingChangingBlobs(Set<Future<Void>> pendingChangingBlobs,
- LocalAssignment pendingChangingBlobsAssignment) {
- return new DynamicState(this.state, this.newAssignment,
- this.container, this.currentAssignment,
- this.pendingLocalization, this.startTime,
- this.pendingDownload, profileActions,
- this.pendingStopProfileActions, this.changingBlobs,
- pendingChangingBlobs,
- pendingChangingBlobsAssignment);
+ dynamicState = new DynamicState(currentAssignment, container, newAssignment);
+ staticState = new StaticState(localizer,
+ ObjectReader.getInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)) * 1000,
+ ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_WORKER_START_TIMEOUT_SECS)) * 1000,
+ ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS)) * 1000,
+ ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_MONITOR_FREQUENCY_SECS)) * 1000,
+ containerLauncher,
+ host,
+ port,
+ iSupervisor,
+ localState,
+ this,
+ metricsExec, metricsProcessor);
+ this.newAssignment.set(dynamicState.newAssignment);
+ if (MachineState.RUNNING == dynamicState.state) {
+ //We are running so we should recover the blobs.
+ staticState.localizer.recoverRunningTopology(currentAssignment, port, this);
+ saveNewAssignment(currentAssignment);
}
- };
+ LOG.info("SLOT {}:{} Starting in state {} - assignment {}", staticState.host, staticState.port, dynamicState.state,
+ dynamicState.currentAssignment);
+ }
//In some cases the new LocalAssignment may be equivalent to the old, but
// It is not equal. In those cases we want to update the current assignment to
@@ -291,55 +158,6 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
return dynamicState;
}
- static class TopoProfileAction {
- public final String topoId;
- public final ProfileRequest request;
-
- public TopoProfileAction(String topoId, ProfileRequest request) {
- this.topoId = topoId;
- this.request = request;
- }
-
- @Override
- public int hashCode() {
- return (37 * topoId.hashCode()) + request.hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (!(other instanceof TopoProfileAction)) {
- return false;
- }
- TopoProfileAction o = (TopoProfileAction) other;
- return topoId.equals(o.topoId) && request.equals(o.request);
- }
-
- @Override
- public String toString() {
- return "{ " + topoId + ": " + request + " }";
- }
- }
-
- /**
- * Holds the information about a blob that is changing.
- */
- static class BlobChanging {
- private final LocalAssignment assignment;
- private final LocallyCachedBlob blob;
- private final GoodToGo.GoodToGoLatch latch;
-
- public BlobChanging(LocalAssignment assignment, LocallyCachedBlob blob, GoodToGo.GoodToGoLatch latch) {
- this.assignment = assignment;
- this.blob = blob;
- this.latch = latch;
- }
-
- @Override
- public String toString() {
- return "BLOB CHANGING " + blob + " " + assignment;
- }
- }
-
@VisibleForTesting
static boolean forSameTopology(LocalAssignment a, LocalAssignment b) {
if (a == null && b == null) {
@@ -378,7 +196,7 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
}
return false;
}
-
+
static DynamicState stateMachineStep(DynamicState dynamicState, StaticState staticState) throws Exception {
LOG.debug("STATE {}", dynamicState.state);
switch (dynamicState.state) {
@@ -399,10 +217,10 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
case WAITING_FOR_BLOB_UPDATE:
return handleWaitingForBlobUpdate(dynamicState, staticState);
default:
- throw new IllegalStateException("Code not ready to handle a state of "+dynamicState.state);
+ throw new IllegalStateException("Code not ready to handle a state of " + dynamicState.state);
}
}
-
+
/**
* Prepare for a new assignment by downloading new required blobs, or going to empty if there is nothing to download.
* PRECONDITION: The slot should be empty
@@ -412,34 +230,34 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
* @throws IOException on any error
*/
static DynamicState prepareForNewAssignmentNoWorkersRunning(DynamicState dynamicState, StaticState staticState) throws IOException {
- assert(dynamicState.container == null);
-
+ assert (dynamicState.container == null);
+
if (dynamicState.newAssignment == null) {
return dynamicState.withState(MachineState.EMPTY);
}
Future<Void> pendingDownload = staticState.localizer.requestDownloadTopologyBlobs(dynamicState.newAssignment,
- staticState.port, staticState.changingCallback);
+ staticState.port, staticState.changingCallback);
return dynamicState.withPendingLocalization(dynamicState.newAssignment, pendingDownload)
- .withState(MachineState.WAITING_FOR_BLOB_LOCALIZATION);
+ .withState(MachineState.WAITING_FOR_BLOB_LOCALIZATION);
}
-
+
/**
* Kill the current container and start downloading what the new assignment needs, if there is a new assignment.
* PRECONDITION: container != null
* @param dynamicState current state
* @param staticState static data
* @return the next state
- * @throws Exception
+ * @throws Exception
*/
static DynamicState killContainerForChangedAssignment(DynamicState dynamicState, StaticState staticState) throws Exception {
- assert(dynamicState.container != null);
+ assert (dynamicState.container != null);
staticState.iSupervisor.killedWorker(staticState.port);
dynamicState.container.kill();
Future<Void> pendingDownload = null;
if (dynamicState.newAssignment != null) {
pendingDownload = staticState.localizer.requestDownloadTopologyBlobs(dynamicState.newAssignment, staticState.port,
- staticState.changingCallback);
+ staticState.changingCallback);
}
dynamicState = drainAllChangingBlobs(dynamicState);
Time.sleep(staticState.killSleepMs);
@@ -454,7 +272,7 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
* @return the next state
*/
private static DynamicState killContainerForChangedBlobs(DynamicState dynamicState, StaticState staticState) throws Exception {
- assert(dynamicState.container != null);
+ assert (dynamicState.container != null);
staticState.iSupervisor.killedWorker(staticState.port);
dynamicState.container.kill();
@@ -469,20 +287,20 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
* @param dynamicState current state
* @param staticState static data
* @return the next state
- * @throws Exception
+ * @throws Exception
*/
static DynamicState killAndRelaunchContainer(DynamicState dynamicState, StaticState staticState) throws Exception {
- assert(dynamicState.container != null);
-
+ assert (dynamicState.container != null);
+
dynamicState.container.kill();
Time.sleep(staticState.killSleepMs);
-
+
//any stop profile actions that hadn't timed out yet, we should restart after the worker is running again.
HashSet<TopoProfileAction> mod = new HashSet<>(dynamicState.profileActions);
mod.addAll(dynamicState.pendingStopProfileActions);
return dynamicState.withState(MachineState.KILL_AND_RELAUNCH).withProfileActions(mod, Collections.emptySet());
}
-
+
/**
* Clean up a container.
* PRECONDITION: All of the processes have died.
@@ -491,11 +309,12 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
* @param nextState the next MachineState to go to.
* @return the next state.
*/
- static DynamicState cleanupCurrentContainer(DynamicState dynamicState, StaticState staticState, MachineState nextState) throws Exception {
- assert(dynamicState.container != null);
- assert(dynamicState.currentAssignment != null);
- assert(dynamicState.container.areAllProcessesDead());
-
+ static DynamicState cleanupCurrentContainer(DynamicState dynamicState, StaticState staticState, MachineState nextState) throws
+ Exception {
+ assert (dynamicState.container != null);
+ assert (dynamicState.currentAssignment != null);
+ assert (dynamicState.container.areAllProcessesDead());
+
dynamicState.container.cleanUp();
staticState.localizer.releaseSlotFor(dynamicState.currentAssignment, staticState.port);
DynamicState ret = dynamicState.withCurrentAssignment(null, null);
@@ -544,13 +363,13 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
}
//Otherwise they will just be replaced
- for (BlobChanging rc: dynamicState.changingBlobs) {
+ for (BlobChanging rc : dynamicState.changingBlobs) {
futures.add(rc.latch.countDown());
}
LOG.debug("found changing blobs {} moving them to pending...", dynamicState.changingBlobs);
return dynamicState.withChangingBlobs(Collections.emptySet())
- .withPendingChangingBlobs(futures, assignment);
+ .withPendingChangingBlobs(futures, assignment);
}
/**
@@ -567,7 +386,7 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
}
HashSet<BlobChanging> savedBlobs = new HashSet<>(dynamicState.changingBlobs.size());
- for (BlobChanging rc: dynamicState.changingBlobs) {
+ for (BlobChanging rc : dynamicState.changingBlobs) {
if (forSameTopology(assignment, rc.assignment)) {
savedBlobs.add(rc);
} else {
@@ -587,10 +406,10 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
* @throws Exception on any error
*/
static DynamicState handleWaitingForBlobLocalization(DynamicState dynamicState, StaticState staticState) throws Exception {
- assert(dynamicState.pendingLocalization != null);
- assert(dynamicState.pendingDownload != null);
- assert(dynamicState.container == null);
-
+ assert (dynamicState.pendingLocalization != null);
+ assert (dynamicState.pendingDownload != null);
+ assert (dynamicState.container == null);
+
//Ignore changes to scheduling while downloading the topology blobs
// We don't support canceling the download through the future yet,
// so to keep everything in sync, just wait
@@ -608,19 +427,20 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
//Scheduling changed
staticState.localizer.releaseSlotFor(dynamicState.pendingLocalization, staticState.port);
return prepareForNewAssignmentNoWorkersRunning(dynamicState.withPendingChangingBlobs(Collections.emptySet(), null),
- staticState);
+ staticState);
}
if (!dynamicState.pendingChangingBlobs.isEmpty()) {
LOG.info("There are pending changes, waiting for them to finish before launching container...");
//We cannot launch the container yet the resources may still be updating
return dynamicState.withState(MachineState.WAITING_FOR_BLOB_UPDATE)
- .withPendingLocalization(null, null);
+ .withPendingLocalization(null, null);
}
dynamicState = updateAssignmentIfNeeded(dynamicState);
numWorkersLaunched.mark();
- Container c = staticState.containerLauncher.launchContainer(staticState.port, dynamicState.pendingLocalization, staticState.localState);
+ Container c =
+ staticState.containerLauncher.launchContainer(staticState.port, dynamicState.pendingLocalization, staticState.localState);
return dynamicState
.withCurrentAssignment(c, dynamicState.pendingLocalization).withState(MachineState.WAITING_FOR_WORKER_START)
.withPendingLocalization(null, null);
@@ -643,8 +463,6 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
}
}
- private static final long ONE_SEC_IN_NANO = TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS);
-
/**
* State Transitions for WAITING_FOR_BLOB_UPDATE state.
*
@@ -667,14 +485,14 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
//We were rescheduled while waiting for the resources to be updated,
// but the container is already not running.
LOG.info("SLOT {}: Assignment Changed from {} to {}", staticState.port,
- dynamicState.currentAssignment, dynamicState.newAssignment);
+ dynamicState.currentAssignment, dynamicState.newAssignment);
if (dynamicState.currentAssignment != null) {
staticState.localizer.releaseSlotFor(dynamicState.currentAssignment, staticState.port);
}
staticState.localizer.releaseSlotFor(dynamicState.pendingChangingBlobsAssignment, staticState.port);
return prepareForNewAssignmentNoWorkersRunning(dynamicState.withCurrentAssignment(null, null)
- .withPendingChangingBlobs(Collections.emptySet(), null),
- staticState);
+ .withPendingChangingBlobs(Collections.emptySet(), null),
+ staticState);
}
dynamicState = filterChangingBlobsFor(dynamicState, dynamicState.pendingChangingBlobsAssignment);
@@ -685,7 +503,7 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
//We only have a set amount of time we can wait for before looping around again
long start = Time.nanoTime();
try {
- for (Future<Void> pending: dynamicState.pendingChangingBlobs) {
+ for (Future<Void> pending : dynamicState.pendingChangingBlobs) {
long now = Time.nanoTime();
long timeLeft = ONE_SEC_IN_NANO - (now - start);
if (timeLeft <= 0) {
@@ -695,7 +513,7 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
}
//All done we can launch the worker now
Container c = staticState.containerLauncher.launchContainer(staticState.port, dynamicState.pendingChangingBlobsAssignment,
- staticState.localState);
+ staticState.localState);
return dynamicState
.withCurrentAssignment(c, dynamicState.pendingChangingBlobsAssignment).withState(MachineState.WAITING_FOR_WORKER_START)
.withPendingChangingBlobs(Collections.emptySet(), null);
@@ -714,13 +532,14 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
* @throws Exception on any error
*/
static DynamicState handleKill(DynamicState dynamicState, StaticState staticState) throws Exception {
- assert(dynamicState.container != null);
- assert(dynamicState.currentAssignment != null);
+ assert (dynamicState.container != null);
+ assert (dynamicState.currentAssignment != null);
if (dynamicState.container.areAllProcessesDead()) {
LOG.info("SLOT {} all processes are dead...", staticState.port);
return cleanupCurrentContainer(dynamicState, staticState,
- dynamicState.pendingLocalization == null ? MachineState.EMPTY : MachineState.WAITING_FOR_BLOB_LOCALIZATION);
+ dynamicState.pendingLocalization ==
+ null ? MachineState.EMPTY : MachineState.WAITING_FOR_BLOB_LOCALIZATION);
}
LOG.warn("SLOT {} force kill and wait...", staticState.port);
@@ -740,9 +559,9 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
* @throws Exception on any error
*/
static DynamicState handleKillAndRelaunch(DynamicState dynamicState, StaticState staticState) throws Exception {
- assert(dynamicState.container != null);
- assert(dynamicState.currentAssignment != null);
-
+ assert (dynamicState.container != null);
+ assert (dynamicState.currentAssignment != null);
+
if (dynamicState.container.areAllProcessesDead()) {
if (equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
dynamicState.container.cleanUpForRestart();
@@ -773,8 +592,8 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
* @throws Exception on any error
*/
private static DynamicState handleKillBlobUpdate(DynamicState dynamicState, StaticState staticState) throws Exception {
- assert(dynamicState.container != null);
- assert(dynamicState.currentAssignment != null);
+ assert (dynamicState.container != null);
+ assert (dynamicState.currentAssignment != null);
//Release things that don't need to wait for us
dynamicState = filterChangingBlobsFor(dynamicState, dynamicState.currentAssignment);
@@ -807,9 +626,9 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
* @throws Exception on any error
*/
static DynamicState handleWaitingForWorkerStart(DynamicState dynamicState, StaticState staticState) throws Exception {
- assert(dynamicState.container != null);
- assert(dynamicState.currentAssignment != null);
-
+ assert (dynamicState.container != null);
+ assert (dynamicState.currentAssignment != null);
+
LSWorkerHeartbeat hb = dynamicState.container.readHeartbeat();
if (hb != null) {
long hbAgeMs = (Time.currentTimeSecs() - hb.get_time_secs()) * 1000;
@@ -817,18 +636,19 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
return dynamicState.withState(MachineState.RUNNING);
}
}
-
+
if (!equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
//We were rescheduled while waiting for the worker to come up
LOG.info("SLOT {}: Assignment Changed from {} to {}", staticState.port, dynamicState.currentAssignment,
- dynamicState.newAssignment);
+ dynamicState.newAssignment);
return killContainerForChangedAssignment(dynamicState, staticState);
}
dynamicState = updateAssignmentIfNeeded(dynamicState);
long timeDiffms = (Time.currentTimeMillis() - dynamicState.startTime);
if (timeDiffms > staticState.firstHbTimeoutMs) {
- LOG.warn("SLOT {}: Container {} failed to launch in {} ms.", staticState.port, dynamicState.container, staticState.firstHbTimeoutMs);
+ LOG.warn("SLOT {}: Container {} failed to launch in {} ms.", staticState.port, dynamicState.container,
+ staticState.firstHbTimeoutMs);
return killAndRelaunchContainer(dynamicState, staticState);
}
@@ -850,12 +670,12 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
* @throws Exception on any error
*/
static DynamicState handleRunning(DynamicState dynamicState, StaticState staticState) throws Exception {
- assert(dynamicState.container != null);
- assert(dynamicState.currentAssignment != null);
-
+ assert (dynamicState.container != null);
+ assert (dynamicState.currentAssignment != null);
+
if (!equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
LOG.info("SLOT {}: Assignment Changed from {} to {}", staticState.port, dynamicState.currentAssignment,
- dynamicState.newAssignment);
+ dynamicState.newAssignment);
//Scheduling changed while running...
return killContainerForChangedAssignment(dynamicState, staticState);
}
@@ -878,7 +698,7 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
LOG.warn("SLOT {}: violated memory limits", staticState.port);
return killAndRelaunchContainer(dynamicState, staticState);
}
-
+
LSWorkerHeartbeat hb = dynamicState.container.readHeartbeat();
if (hb == null) {
numWorkersKilledHBNull.mark();
@@ -887,14 +707,14 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
// worker that never came up.
return killAndRelaunchContainer(dynamicState, staticState);
}
-
+
long timeDiffMs = (Time.currentTimeSecs() - hb.get_time_secs()) * 1000;
if (timeDiffMs > staticState.hbTimeoutMs) {
numWorkersKilledHBTimeout.mark();
LOG.warn("SLOT {}: HB is too old {} > {}", staticState.port, timeDiffMs, staticState.hbTimeoutMs);
return killAndRelaunchContainer(dynamicState, staticState);
}
-
+
//The worker is up and running check for profiling requests
if (!dynamicState.profileActions.isEmpty()) {
HashSet<TopoProfileAction> mod = new HashSet<>(dynamicState.profileActions);
@@ -943,114 +763,37 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
}
dynamicState = dynamicState.withProfileActions(mod, modPending);
}
-
- dynamicState.container.processMetrics(staticState.metricsExec, staticState.metricsProcessor);
-
- Time.sleep(staticState.monitorFreqMs);
- return dynamicState;
- }
-
- static DynamicState handleEmpty(DynamicState dynamicState, StaticState staticState) throws InterruptedException, IOException {
- assert dynamicState.changingBlobs.isEmpty();
- assert dynamicState.pendingChangingBlobsAssignment == null;
- if (!equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
- return prepareForNewAssignmentNoWorkersRunning(dynamicState, staticState);
- }
- dynamicState = updateAssignmentIfNeeded(dynamicState);
-
- //Both assignments are null, just wait
- if (dynamicState.profileActions != null && !dynamicState.profileActions.isEmpty()) {
- //Nothing is scheduled here so throw away all of the profileActions
- LOG.warn("Dropping {} no topology is running", dynamicState.profileActions);
- dynamicState = dynamicState.withProfileActions(Collections.emptySet(), Collections.emptySet());
- }
- //Drop the change notifications we are not running anything right now
- dynamicState = drainAllChangingBlobs(dynamicState);
- Time.sleep(1000);
- return dynamicState;
- }
-
- private final AtomicReference<LocalAssignment> newAssignment = new AtomicReference<>();
- private final AtomicReference<Set<TopoProfileAction>> profiling = new AtomicReference<>(new HashSet<>());
- private final BlockingQueue<BlobChanging> changingBlobs = new LinkedBlockingQueue<>();
- private final StaticState staticState;
- private final IStormClusterState clusterState;
- private volatile boolean done = false;
- private volatile DynamicState dynamicState;
- private final AtomicReference<Map<Long, LocalAssignment>> cachedCurrentAssignments;
- private final OnlyLatestExecutor<Integer> metricsExec;
-
- public Slot(AsyncLocalizer localizer, Map<String, Object> conf,
- ContainerLauncher containerLauncher, String host,
- int port, LocalState localState,
- IStormClusterState clusterState,
- ISupervisor iSupervisor,
- AtomicReference<Map<Long, LocalAssignment>> cachedCurrentAssignments,
- OnlyLatestExecutor<Integer> metricsExec,
- WorkerMetricsProcessor metricsProcessor) throws Exception {
- super("SLOT_"+port);
- this.metricsExec = metricsExec;
-
- this.cachedCurrentAssignments = cachedCurrentAssignments;
- this.clusterState = clusterState;
- Map<Integer, LocalAssignment> assignments = localState.getLocalAssignmentsMap();
- LocalAssignment currentAssignment = null;
- if (assignments != null) {
- currentAssignment = assignments.get(port);
- }
- Container container = null;
- if (currentAssignment != null) {
- try {
- // For now we do not make a transaction when removing a topology assignment from local, an overdue
- // assignment may be left on local disk.
- // So we should check if the local disk assignment is valid when initializing:
- // if topology files does not exist, the worker[possibly alive] will be reassigned if it is timed-out;
- // if topology files exist but the topology id is invalid, just let Supervisor make a sync;
- // if topology files exist and topology files is valid, recover the container.
- if (ClientSupervisorUtils.doRequiredTopoFilesExist(conf, currentAssignment.get_topology_id())) {
- container = containerLauncher.recoverContainer(port, currentAssignment, localState);
- } else {
- // Make the assignment null to let slot clean up the disk assignment.
- currentAssignment = null;
- }
- } catch (ContainerRecoveryException e) {
- //We could not recover container will be null.
- }
- }
-
- LocalAssignment newAssignment = currentAssignment;
- if (currentAssignment != null && container == null) {
- currentAssignment = null;
- //Assigned something but it is not running
+
+ dynamicState.container.processMetrics(staticState.metricsExec, staticState.metricsProcessor);
+
+ Time.sleep(staticState.monitorFreqMs);
+ return dynamicState;
+ }
+
+ static DynamicState handleEmpty(DynamicState dynamicState, StaticState staticState) throws InterruptedException, IOException {
+ assert dynamicState.changingBlobs.isEmpty();
+ assert dynamicState.pendingChangingBlobsAssignment == null;
+ if (!equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
+ return prepareForNewAssignmentNoWorkersRunning(dynamicState, staticState);
}
-
- dynamicState = new DynamicState(currentAssignment, container, newAssignment);
- staticState = new StaticState(localizer,
- ObjectReader.getInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)) * 1000,
- ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_WORKER_START_TIMEOUT_SECS)) * 1000,
- ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS)) * 1000,
- ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_MONITOR_FREQUENCY_SECS)) * 1000,
- containerLauncher,
- host,
- port,
- iSupervisor,
- localState,
- this,
- metricsExec, metricsProcessor);
- this.newAssignment.set(dynamicState.newAssignment);
- if (MachineState.RUNNING == dynamicState.state) {
- //We are running so we should recover the blobs.
- staticState.localizer.recoverRunningTopology(currentAssignment, port, this);
- saveNewAssignment(currentAssignment);
+ dynamicState = updateAssignmentIfNeeded(dynamicState);
+
+ //Both assignments are null, just wait
+ if (dynamicState.profileActions != null && !dynamicState.profileActions.isEmpty()) {
+ //Nothing is scheduled here so throw away all of the profileActions
+ LOG.warn("Dropping {} no topology is running", dynamicState.profileActions);
+ dynamicState = dynamicState.withProfileActions(Collections.emptySet(), Collections.emptySet());
}
- LOG.info("SLOT {}:{} Starting in state {} - assignment {}", staticState.host, staticState.port, dynamicState.state,
- dynamicState.currentAssignment);
+ //Drop the change notifications we are not running anything right now
+ dynamicState = drainAllChangingBlobs(dynamicState);
+ Time.sleep(1000);
+ return dynamicState;
}
-
+
public MachineState getMachineState() {
return dynamicState.state;
}
-
+
/**
* Set a new assignment asynchronously.
* @param newAssignment the new assignment for this slot to run, null to run nothing
@@ -1072,7 +815,7 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
public void addProfilerActions(Set<TopoProfileAction> actions) {
if (actions != null) {
- while(true) {
+ while (true) {
Set<TopoProfileAction> orig = profiling.get();
Set<TopoProfileAction> newActions = new HashSet<>(orig);
newActions.addAll(actions);
@@ -1082,7 +825,7 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
}
}
}
-
+
public String getWorkerId() {
String workerId = null;
Container c = dynamicState.container;
@@ -1091,9 +834,9 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
}
return workerId;
}
-
+
private void saveNewAssignment(LocalAssignment assignment) {
- synchronized(staticState.localState) {
+ synchronized (staticState.localState) {
Map<Integer, LocalAssignment> assignments = staticState.localState.getLocalAssignmentsMap();
if (assignments == null) {
assignments = new HashMap<>();
@@ -1118,10 +861,10 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
}
} while (!cachedCurrentAssignments.compareAndSet(orig, update));
}
-
+
public void run() {
try {
- while(!done) {
+ while (!done) {
Set<TopoProfileAction> origProfileActions = new HashSet<>(profiling.get());
Set<TopoProfileAction> removed = new HashSet<>(origProfileActions);
@@ -1132,7 +875,7 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
Iterator<BlobChanging> it = changingResourcesToHandle.iterator();
//Remove/Clean up changed requests that are not for us
- while(it.hasNext()) {
+ while (it.hasNext()) {
BlobChanging rc = it.next();
if (!forSameTopology(rc.assignment, dynamicState.currentAssignment) &&
!forSameTopology(rc.assignment, dynamicState.newAssignment)) {
@@ -1144,16 +887,17 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
DynamicState nextState =
stateMachineStep(dynamicState.withNewAssignment(newAssignment.get())
- .withProfileActions(origProfileActions, dynamicState.pendingStopProfileActions)
- .withChangingBlobs(changingResourcesToHandle), staticState);
+ .withProfileActions(origProfileActions, dynamicState.pendingStopProfileActions)
+ .withChangingBlobs(changingResourcesToHandle), staticState);
if (LOG.isDebugEnabled() || dynamicState.state != nextState.state) {
LOG.info("STATE {} -> {}", dynamicState, nextState);
}
//Save the current state for recovery
if ((nextState.currentAssignment != null && !nextState.currentAssignment.equals(dynamicState.currentAssignment)) ||
- (dynamicState.currentAssignment != null && !dynamicState.currentAssignment.equals(nextState.currentAssignment))) {
- LOG.info("SLOT {}: Changing current assignment from {} to {}", staticState.port, dynamicState.currentAssignment, nextState.currentAssignment);
+ (dynamicState.currentAssignment != null && !dynamicState.currentAssignment.equals(nextState.currentAssignment))) {
+ LOG.info("SLOT {}: Changing current assignment from {} to {}", staticState.port, dynamicState.currentAssignment,
+ nextState.currentAssignment);
saveNewAssignment(nextState.currentAssignment);
}
@@ -1167,11 +911,11 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
saveNewAssignment(nextState.newAssignment);
nextState = nextState.withCurrentAssignment(nextState.container, nextState.newAssignment);
}
-
+
// clean up the profiler actions that are not being processed
removed.removeAll(dynamicState.profileActions);
removed.removeAll(dynamicState.pendingStopProfileActions);
- for (TopoProfileAction action: removed) {
+ for (TopoProfileAction action : removed) {
try {
clusterState.deleteTopologyProfileRequests(action.topoId, action.request);
} catch (Exception e) {
@@ -1200,4 +944,259 @@ public class Slot extends Thread implements AutoCloseable, BlobChangingCallback
this.interrupt();
this.join();
}
+
+ static enum MachineState {
+ EMPTY,
+ RUNNING,
+ WAITING_FOR_WORKER_START,
+ KILL_AND_RELAUNCH,
+ KILL,
+ KILL_BLOB_UPDATE,
+ WAITING_FOR_BLOB_LOCALIZATION,
+ WAITING_FOR_BLOB_UPDATE;
+ }
+
+ static class StaticState {
+ public final AsyncLocalizer localizer;
+ public final long hbTimeoutMs;
+ public final long firstHbTimeoutMs;
+ public final long killSleepMs;
+ public final long monitorFreqMs;
+ public final ContainerLauncher containerLauncher;
+ public final int port;
+ public final String host;
+ public final ISupervisor iSupervisor;
+ public final LocalState localState;
+ public final BlobChangingCallback changingCallback;
+ public final OnlyLatestExecutor<Integer> metricsExec;
+ public final WorkerMetricsProcessor metricsProcessor;
+
+ StaticState(AsyncLocalizer localizer, long hbTimeoutMs, long firstHbTimeoutMs,
+ long killSleepMs, long monitorFreqMs,
+ ContainerLauncher containerLauncher, String host, int port,
+ ISupervisor iSupervisor, LocalState localState,
+ BlobChangingCallback changingCallback,
+ OnlyLatestExecutor<Integer> metricsExec,
+ WorkerMetricsProcessor metricsProcessor) {
+ this.localizer = localizer;
+ this.hbTimeoutMs = hbTimeoutMs;
+ this.firstHbTimeoutMs = firstHbTimeoutMs;
+ this.containerLauncher = containerLauncher;
+ this.killSleepMs = killSleepMs;
+ this.monitorFreqMs = monitorFreqMs;
+ this.host = host;
+ this.port = port;
+ this.iSupervisor = iSupervisor;
+ this.localState = localState;
+ this.changingCallback = changingCallback;
+ this.metricsExec = metricsExec;
+ this.metricsProcessor = metricsProcessor;
+ }
+ }
+
+ static class DynamicState {
+ public final MachineState state;
+ public final LocalAssignment newAssignment;
+ public final LocalAssignment currentAssignment;
+ public final Container container;
+ public final LocalAssignment pendingLocalization;
+ public final Future<Void> pendingDownload;
+ public final Set<TopoProfileAction> profileActions;
+ public final Set<TopoProfileAction> pendingStopProfileActions;
+ public final Set<BlobChanging> changingBlobs;
+ public final LocalAssignment pendingChangingBlobsAssignment;
+ public final Set<Future<Void>> pendingChangingBlobs;
+
+ /**
+ * The last time that WAITING_FOR_WORKER_START, KILL, or KILL_AND_RELAUNCH were entered into.
+ */
+ public final long startTime;
+
+ public DynamicState(final LocalAssignment currentAssignment, Container container, final LocalAssignment newAssignment) {
+ this.currentAssignment = currentAssignment;
+ this.container = container;
+ if ((currentAssignment == null) ^ (container == null)) {
+ throw new IllegalArgumentException("Container and current assignment must both be null, or neither can be null");
+ }
+
+ if (currentAssignment == null) {
+ state = MachineState.EMPTY;
+ } else {
+ state = MachineState.RUNNING;
+ }
+
+ this.startTime = System.currentTimeMillis();
+ this.newAssignment = newAssignment;
+ this.pendingLocalization = null;
+ this.pendingDownload = null;
+ this.profileActions = Collections.emptySet();
+ this.pendingStopProfileActions = Collections.emptySet();
+ this.changingBlobs = Collections.emptySet();
+ this.pendingChangingBlobsAssignment = null;
+ this.pendingChangingBlobs = Collections.emptySet();
+ }
+
+ public DynamicState(final MachineState state, final LocalAssignment newAssignment,
+ final Container container, final LocalAssignment currentAssignment,
+ final LocalAssignment pendingLocalization, final long startTime,
+ final Future<Void> pendingDownload, final Set<TopoProfileAction> profileActions,
+ final Set<TopoProfileAction> pendingStopProfileActions,
+ final Set<BlobChanging> changingBlobs,
+ final Set<Future<Void>> pendingChangingBlobs, final LocalAssignment pendingChaningBlobsAssignment) {
+ assert pendingChangingBlobs != null;
+ assert !(pendingChangingBlobs.isEmpty() ^ (pendingChaningBlobsAssignment == null));
+ this.state = state;
+ this.newAssignment = newAssignment;
+ this.currentAssignment = currentAssignment;
+ this.container = container;
+ this.pendingLocalization = pendingLocalization;
+ this.startTime = startTime;
+ this.pendingDownload = pendingDownload;
+ this.profileActions = profileActions;
+ this.pendingStopProfileActions = pendingStopProfileActions;
+ this.changingBlobs = changingBlobs;
+ this.pendingChangingBlobs = pendingChangingBlobs;
+ this.pendingChangingBlobsAssignment = pendingChaningBlobsAssignment;
+ }
+
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(state);
+ sb.append(" msInState: ");
+ sb.append(Time.currentTimeMillis() - startTime);
+ if (container != null) {
+ sb.append(" ");
+ sb.append(container);
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Set the new assignment for the state. This should never be called from within the state machine.
+ * It is an input from outside.
+ * @param newAssignment the new assignment to set
+ * @return the updated DynamicState.
+ */
+ public DynamicState withNewAssignment(LocalAssignment newAssignment) {
+ return new DynamicState(this.state, newAssignment,
+ this.container, this.currentAssignment,
+ this.pendingLocalization, this.startTime,
+ this.pendingDownload, this.profileActions,
+ this.pendingStopProfileActions, this.changingBlobs,
+ this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
+ }
+
+ public DynamicState withPendingLocalization(LocalAssignment pendingLocalization, Future<Void> pendingDownload) {
+ return new DynamicState(this.state, this.newAssignment,
+ this.container, this.currentAssignment,
+ pendingLocalization, this.startTime,
+ pendingDownload, this.profileActions,
+ this.pendingStopProfileActions, this.changingBlobs,
+ this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
+ }
+
+ public DynamicState withPendingLocalization(Future<Void> pendingDownload) {
+ return withPendingLocalization(this.pendingLocalization, pendingDownload);
+ }
+
+ public DynamicState withState(final MachineState state) {
+ long newStartTime = Time.currentTimeMillis();
+ return new DynamicState(state, this.newAssignment,
+ this.container, this.currentAssignment,
+ this.pendingLocalization, newStartTime,
+ this.pendingDownload, this.profileActions,
+ this.pendingStopProfileActions, this.changingBlobs,
+ this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
+ }
+
+ public DynamicState withCurrentAssignment(final Container container, final LocalAssignment currentAssignment) {
+ return new DynamicState(this.state, this.newAssignment,
+ container, currentAssignment,
+ this.pendingLocalization, this.startTime,
+ this.pendingDownload, this.profileActions,
+ this.pendingStopProfileActions, this.changingBlobs,
+ this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
+ }
+
+ public DynamicState withProfileActions(Set<TopoProfileAction> profileActions, Set<TopoProfileAction> pendingStopProfileActions) {
+ return new DynamicState(this.state, this.newAssignment,
+ this.container, this.currentAssignment,
+ this.pendingLocalization, this.startTime,
+ this.pendingDownload, profileActions,
+ pendingStopProfileActions, this.changingBlobs,
+ this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
+ }
+
+ public DynamicState withChangingBlobs(Set<BlobChanging> changingBlobs) {
+ if (changingBlobs == this.changingBlobs) {
+ return this;
+ }
+ return new DynamicState(this.state, this.newAssignment,
+ this.container, this.currentAssignment,
+ this.pendingLocalization, this.startTime,
+ this.pendingDownload, profileActions,
+ this.pendingStopProfileActions, changingBlobs,
+ this.pendingChangingBlobs, this.pendingChangingBlobsAssignment);
+ }
+
+ public DynamicState withPendingChangingBlobs(Set<Future<Void>> pendingChangingBlobs,
+ LocalAssignment pendingChangingBlobsAssignment) {
+ return new DynamicState(this.state, this.newAssignment,
+ this.container, this.currentAssignment,
+ this.pendingLocalization, this.startTime,
+ this.pendingDownload, profileActions,
+ this.pendingStopProfileActions, this.changingBlobs,
+ pendingChangingBlobs,
+ pendingChangingBlobsAssignment);
+ }
+ }
+
+ static class TopoProfileAction {
+ public final String topoId;
+ public final ProfileRequest request;
+
+ public TopoProfileAction(String topoId, ProfileRequest request) {
+ this.topoId = topoId;
+ this.request = request;
+ }
+
+ @Override
+ public int hashCode() {
+ return (37 * topoId.hashCode()) + request.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof TopoProfileAction)) {
+ return false;
+ }
+ TopoProfileAction o = (TopoProfileAction) other;
+ return topoId.equals(o.topoId) && request.equals(o.request);
+ }
+
+ @Override
+ public String toString() {
+ return "{ " + topoId + ": " + request + " }";
+ }
+ }
+
+ /**
+ * Holds the information about a blob that is changing.
+ */
+ static class BlobChanging {
+ private final LocalAssignment assignment;
+ private final LocallyCachedBlob blob;
+ private final GoodToGo.GoodToGoLatch latch;
+
+ public BlobChanging(LocalAssignment assignment, LocallyCachedBlob blob, GoodToGo.GoodToGoLatch latch) {
+ this.assignment = assignment;
+ this.blob = blob;
+ this.latch = latch;
+ }
+
+ @Override
+ public String toString() {
+ return "BLOB CHANGING " + blob + " " + assignment;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java
index 1332a21..0730957 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java
@@ -1,32 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.daemon.supervisor;
-import org.apache.storm.DaemonConfig;
-import org.apache.storm.scheduler.ISupervisor;
-import org.apache.storm.utils.Utils;
-import org.apache.storm.utils.LocalState;
+package org.apache.storm.daemon.supervisor;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collection;
import java.util.Map;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
public class StandaloneSupervisor implements ISupervisor {
private String supervisorId;
@@ -79,7 +73,7 @@ public class StandaloneSupervisor implements ISupervisor {
}
- public String generateSupervisorId(){
+ public String generateSupervisorId() {
String extraPart = "";
try {
extraPart = "-" + InetAddress.getLocalHost().getHostAddress();
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Supervisor.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Supervisor.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Supervisor.java
index 4e5b861..c445383 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Supervisor.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Supervisor.java
@@ -8,7 +8,7 @@
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,6 +18,7 @@
package org.apache.storm.daemon.supervisor;
+import com.google.common.annotations.VisibleForTesting;
import java.io.File;
import java.io.IOException;
import java.net.BindException;
@@ -31,8 +32,6 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicReference;
-
-import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.FileUtils;
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
@@ -66,8 +65,6 @@ import org.apache.storm.security.auth.ReqContext;
import org.apache.storm.security.auth.ThriftConnectionType;
import org.apache.storm.security.auth.ThriftServer;
import org.apache.storm.utils.ConfigUtils;
-import org.apache.storm.utils.ServerConfigUtils;
-import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.LocalState;
import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.ServerConfigUtils;
@@ -76,7 +73,6 @@ import org.apache.storm.utils.Utils;
import org.apache.storm.utils.VersionInfo;
import org.apache.thrift.TException;
import org.apache.thrift.TProcessor;
-import org.apache.zookeeper.data.ACL;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -85,7 +81,6 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
private final Map<String, Object> conf;
private final IContext sharedContext;
private final IAuthorizer authorizationHandler;
- private volatile boolean active;
private final ISupervisor iSupervisor;
private final Utils.UptimeComputer upTime;
private final String stormVersion;
@@ -105,12 +100,13 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
// to really make this work well.
private final ExecutorService heartbeatExecutor;
private final AsyncLocalizer asyncLocalizer;
+ private volatile boolean active;
private EventManager eventManager;
private ReadClusterState readState;
private ThriftServer thriftServer;
//used for local cluster heartbeating
private Nimbus.Iface localNimbus;
-
+
private Supervisor(ISupervisor iSupervisor)
throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException {
this(Utils.readStormConfig(), null, iSupervisor);
@@ -118,13 +114,14 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
/**
* Constructor for supervisor daemon.
- * @param conf config
+ *
+ * @param conf config
* @param sharedContext {@link IContext}
- * @param iSupervisor {@link ISupervisor}
+ * @param iSupervisor {@link ISupervisor}
* @throws IOException
*/
public Supervisor(Map<String, Object> conf, IContext sharedContext, ISupervisor iSupervisor)
- throws IOException, IllegalAccessException, ClassNotFoundException, InstantiationException{
+ throws IOException, IllegalAccessException, ClassNotFoundException, InstantiationException {
this.conf = conf;
this.iSupervisor = iSupervisor;
this.active = true;
@@ -133,17 +130,17 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
this.sharedContext = sharedContext;
this.heartbeatExecutor = Executors.newFixedThreadPool(1);
this.authorizationHandler = StormCommon.mkAuthorizationHandler(
- (String) conf.get(DaemonConfig.SUPERVISOR_AUTHORIZER), conf);
+ (String) conf.get(DaemonConfig.SUPERVISOR_AUTHORIZER), conf);
if (authorizationHandler == null && conf.get(DaemonConfig.NIMBUS_AUTHORIZER) != null) {
throw new IllegalStateException("It looks like authorization is turned on for nimbus but not for the "
- + "supervisor....");
+ + "supervisor....");
}
-
+
iSupervisor.prepare(conf, ServerConfigUtils.supervisorIsupervisorDir(conf));
try {
this.stormClusterState = ClusterUtils.mkStormClusterState(conf,
- new ClusterStateContext(DaemonType.SUPERVISOR, conf));
+ new ClusterStateContext(DaemonType.SUPERVISOR, conf));
} catch (Exception e) {
LOG.error("supervisor can't create stormClusterState");
throw Utils.wrapInRuntime(e);
@@ -175,6 +172,18 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
}
/**
+ * supervisor daemon enter entrance.
+ *
+ * @param args
+ */
+ public static void main(String[] args) throws Exception {
+ Utils.setupDefaultUncaughtExceptionHandler();
+ @SuppressWarnings("resource")
+ Supervisor instance = new Supervisor(new StandaloneSupervisor());
+ instance.launchDaemon();
+ }
+
+ /**
* Get the executor service that is supposed to be used for heart-beats.
*/
public ExecutorService getHeartbeatExecutor() {
@@ -184,7 +193,7 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
public String getId() {
return supervisorId;
}
-
+
IContext getSharedContext() {
return sharedContext;
}
@@ -232,11 +241,11 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
public AtomicReference<Map<Long, LocalAssignment>> getCurrAssignment() {
return currAssignment;
}
-
+
AsyncLocalizer getAsyncLocalizer() {
return asyncLocalizer;
}
-
+
EventManager getEventManger() {
return eventManager;
}
@@ -245,14 +254,14 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
return this;
}
+ public Nimbus.Iface getLocalNimbus() {
+ return this.localNimbus;
+ }
+
public void setLocalNimbus(Nimbus.Iface nimbus) {
this.localNimbus = nimbus;
}
- public Nimbus.Iface getLocalNimbus() {
- return this.localNimbus;
- }
-
/**
* Launch the supervisor.
*/
@@ -276,7 +285,7 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
// This isn't strictly necessary, but it doesn't hurt and ensures that the machine stays up
// to date even if callbacks don't all work exactly right
eventTimer.scheduleRecurring(0, 10,
- new EventManagerPushCallback(new SynchronizeAssignments(this, null, readState), eventManager));
+ new EventManagerPushCallback(new SynchronizeAssignments(this, null, readState), eventManager));
// supervisor health check
eventTimer.scheduleRecurring(30, 30, new SupervisorHealthCheck(this));
@@ -323,7 +332,7 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
@VisibleForTesting
public void checkAuthorization(String topoName, Map<String, Object> topoConf, String operation, ReqContext context)
- throws AuthorizationException {
+ throws AuthorizationException {
IAuthorizer aclHandler = authorizationHandler;
if (context == null) {
context = ReqContext.context();
@@ -337,19 +346,19 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
if (context.isImpersonating()) {
LOG.warn("principal: {} is trying to impersonate principal: {}", context.realPrincipal(),
- context.principal());
+ context.principal());
throw new AuthorizationException("Supervisor does not support impersonation");
}
if (aclHandler != null) {
if (!aclHandler.permit(context, operation, checkConf)) {
ThriftAccessLogger.logAccess(context.requestID(), context.remoteAddress(), context.principal(),
- operation, topoName, "access-denied");
- throw new AuthorizationException( operation + (topoName != null ? " on topology " + topoName : "") +
- " is not authorized");
+ operation, topoName, "access-denied");
+ throw new AuthorizationException(operation + (topoName != null ? " on topology " + topoName : "") +
+ " is not authorized");
} else {
ThriftAccessLogger.logAccess(context.requestID(), context.remoteAddress(), context.principal(),
- operation, topoName, "access-granted");
+ operation, topoName, "access-granted");
}
}
}
@@ -366,58 +375,59 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
}
TProcessor processor = new org.apache.storm.generated.Supervisor.Processor(
- new org.apache.storm.generated.Supervisor.Iface() {
- @Override
- public void sendSupervisorAssignments(SupervisorAssignments assignments)
- throws AuthorizationException, TException {
- checkAuthorization("sendSupervisorAssignments");
- LOG.info("Got an assignments from master, will start to sync with assignments: {}", assignments);
- SynchronizeAssignments syn = new SynchronizeAssignments(getSupervisor(), assignments,
- getReadClusterState());
- getEventManger().add(syn);
- }
+ new org.apache.storm.generated.Supervisor.Iface() {
+ @Override
+ public void sendSupervisorAssignments(SupervisorAssignments assignments)
+ throws AuthorizationException, TException {
+ checkAuthorization("sendSupervisorAssignments");
+ LOG.info("Got an assignments from master, will start to sync with assignments: {}", assignments);
+ SynchronizeAssignments syn = new SynchronizeAssignments(getSupervisor(), assignments,
+ getReadClusterState());
+ getEventManger().add(syn);
+ }
- @Override
- public Assignment getLocalAssignmentForStorm(String id)
- throws NotAliveException, AuthorizationException, TException {
- Map<String, Object> topoConf = null;
- try {
- topoConf = ConfigUtils.readSupervisorStormConf(conf, id);
- } catch (IOException e) {
- LOG.warn("Topology config is not localized yet...");
- }
- checkAuthorization(id, topoConf, "getLocalAssignmentForStorm");
- Assignment assignment = getStormClusterState().assignmentInfo(id, null);
- if (null == assignment) {
- throw new NotAliveException("No local assignment assigned for storm: "
- + id
- + " for node: "
- + getHostName());
- }
- return assignment;
+ @Override
+ public Assignment getLocalAssignmentForStorm(String id)
+ throws NotAliveException, AuthorizationException, TException {
+ Map<String, Object> topoConf = null;
+ try {
+ topoConf = ConfigUtils.readSupervisorStormConf(conf, id);
+ } catch (IOException e) {
+ LOG.warn("Topology config is not localized yet...");
}
+ checkAuthorization(id, topoConf, "getLocalAssignmentForStorm");
+ Assignment assignment = getStormClusterState().assignmentInfo(id, null);
+ if (null == assignment) {
+ throw new NotAliveException("No local assignment assigned for storm: "
+ + id
+ + " for node: "
+ + getHostName());
+ }
+ return assignment;
+ }
- @Override
- public void sendSupervisorWorkerHeartbeat(SupervisorWorkerHeartbeat heartbeat)
- throws AuthorizationException, NotAliveException, TException {
- // do nothing except validate heartbeat for now.
- String id = heartbeat.get_storm_id();
- Map<String, Object> topoConf = null;
- try {
- topoConf = ConfigUtils.readSupervisorStormConf(conf, id);
- } catch (IOException e) {
- LOG.warn("Topology config is not localized yet...");
- throw new NotAliveException(id + " does not appear to be alive, you should probably exit");
- }
- checkAuthorization(id, topoConf, "sendSupervisorWorkerHeartbeat");
+ @Override
+ public void sendSupervisorWorkerHeartbeat(SupervisorWorkerHeartbeat heartbeat)
+ throws AuthorizationException, NotAliveException, TException {
+ // do nothing except validate heartbeat for now.
+ String id = heartbeat.get_storm_id();
+ Map<String, Object> topoConf = null;
+ try {
+ topoConf = ConfigUtils.readSupervisorStormConf(conf, id);
+ } catch (IOException e) {
+ LOG.warn("Topology config is not localized yet...");
+ throw new NotAliveException(id + " does not appear to be alive, you should probably exit");
}
- });
+ checkAuthorization(id, topoConf, "sendSupervisorWorkerHeartbeat");
+ }
+ });
this.thriftServer = new ThriftServer(conf, processor, ThriftConnectionType.SUPERVISOR);
this.thriftServer.serve();
}
/**
* Used for local cluster assignments distribution.
+ *
* @param assignments {@link SupervisorAssignments}
*/
public void sendSupervisorAssignments(SupervisorAssignments assignments) {
@@ -438,7 +448,7 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
}
});
}
-
+
@Override
public void close() {
try {
@@ -455,14 +465,14 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
}
asyncLocalizer.close();
getStormClusterState().disconnect();
- if(thriftServer != null) {
+ if (thriftServer != null) {
this.thriftServer.stop();
}
} catch (Exception e) {
LOG.error("Error Shutting down", e);
}
}
-
+
void killWorkers(Collection<String> workerIds, ContainerLauncher launcher) throws InterruptedException, IOException {
HashSet<Killable> containers = new HashSet<>();
for (String workerId : workerIds) {
@@ -482,14 +492,14 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
if (!containers.isEmpty()) {
Time.sleepSecs(shutdownSleepSecs);
}
- for (Killable k: containers) {
+ for (Killable k : containers) {
try {
k.forceKill();
long start = Time.currentTimeMillis();
while (!k.areAllProcessesDead()) {
if ((Time.currentTimeMillis() - start) > 10_000) {
- throw new RuntimeException("Giving up on killing " + k
- + " after " + (Time.currentTimeMillis() - start) + " ms");
+ throw new RuntimeException("Giving up on killing " + k
+ + " after " + (Time.currentTimeMillis() - start) + " ms");
}
Time.sleep(100);
k.forceKill();
@@ -507,7 +517,7 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
} else {
try {
ContainerLauncher launcher = ContainerLauncher.make(getConf(), getId(), getThriftServerPort(),
- getSharedContext());
+ getSharedContext());
killWorkers(SupervisorUtils.supervisorWorkerIds(conf), launcher);
} catch (Exception e) {
throw Utils.wrapInRuntime(e);
@@ -522,20 +532,8 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
}
return heartbeatTimer.isTimerWaiting()
- && workerHeartbeatTimer.isTimerWaiting()
- && eventTimer.isTimerWaiting()
- && eventManager.waiting();
- }
-
- /**
- * supervisor daemon enter entrance.
- *
- * @param args
- */
- public static void main(String[] args) throws Exception {
- Utils.setupDefaultUncaughtExceptionHandler();
- @SuppressWarnings("resource")
- Supervisor instance = new Supervisor(new StandaloneSupervisor());
- instance.launchDaemon();
+ && workerHeartbeatTimer.isTimerWaiting()
+ && eventTimer.isTimerWaiting()
+ && eventManager.waiting();
}
}
[17/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
index 6581e04..6d40db9 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java
@@ -24,7 +24,6 @@ import com.codahale.metrics.Meter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
-
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -57,7 +56,6 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.security.auth.Subject;
-
import org.apache.curator.framework.CuratorFramework;
import org.apache.storm.Config;
import org.apache.storm.Constants;
@@ -144,11 +142,11 @@ import org.apache.storm.metric.StormMetricsRegistry;
import org.apache.storm.metric.api.DataPoint;
import org.apache.storm.metric.api.IClusterMetricsConsumer;
import org.apache.storm.metric.api.IClusterMetricsConsumer.ClusterInfo;
-import org.apache.storm.nimbus.AssignmentDistributionService;
import org.apache.storm.metricstore.AggLevel;
import org.apache.storm.metricstore.Metric;
import org.apache.storm.metricstore.MetricStore;
import org.apache.storm.metricstore.MetricStoreConfig;
+import org.apache.storm.nimbus.AssignmentDistributionService;
import org.apache.storm.nimbus.DefaultTopologyValidator;
import org.apache.storm.nimbus.ILeaderElector;
import org.apache.storm.nimbus.ITopologyActionNotifierPlugin;
@@ -211,8 +209,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Nimbus implements Iface, Shutdownable, DaemonCommon {
+ @VisibleForTesting
+ public static final List<ACL> ZK_ACLS = Arrays.asList(ZooDefs.Ids.CREATOR_ALL_ACL.get(0),
+ new ACL(ZooDefs.Perms.READ | ZooDefs.Perms.CREATE, ZooDefs.Ids.ANYONE_ID_UNSAFE));
+ public static final SimpleVersion MIN_VERSION_SUPPORT_RPC_HEARTBEAT = new SimpleVersion("2.0.0");
private static final Logger LOG = LoggerFactory.getLogger(Nimbus.class);
-
// Metrics
private static final Meter submitTopologyWithOptsCalls = StormMetricsRegistry.registerMeter("nimbus:num-submitTopologyWithOpts-calls");
private static final Meter submitTopologyCalls = StormMetricsRegistry.registerMeter("nimbus:num-submitTopology-calls");
@@ -224,7 +225,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
private static final Meter debugCalls = StormMetricsRegistry.registerMeter("nimbus:num-debug-calls");
private static final Meter setWorkerProfilerCalls = StormMetricsRegistry.registerMeter("nimbus:num-setWorkerProfiler-calls");
private static final Meter getComponentPendingProfileActionsCalls = StormMetricsRegistry.registerMeter(
- "nimbus:num-getComponentPendingProfileActions-calls");
+ "nimbus:num-getComponentPendingProfileActions-calls");
private static final Meter setLogConfigCalls = StormMetricsRegistry.registerMeter("nimbus:num-setLogConfig-calls");
private static final Meter uploadNewCredentialsCalls = StormMetricsRegistry.registerMeter("nimbus:num-uploadNewCredentials-calls");
private static final Meter beginFileUploadCalls = StormMetricsRegistry.registerMeter("nimbus:num-beginFileUpload-calls");
@@ -241,59 +242,47 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
private static final Meter getLeaderCalls = StormMetricsRegistry.registerMeter("nimbus:num-getLeader-calls");
private static final Meter isTopologyNameAllowedCalls = StormMetricsRegistry.registerMeter("nimbus:num-isTopologyNameAllowed-calls");
private static final Meter getTopologyInfoWithOptsCalls = StormMetricsRegistry.registerMeter(
- "nimbus:num-getTopologyInfoWithOpts-calls");
+ "nimbus:num-getTopologyInfoWithOpts-calls");
private static final Meter getTopologyInfoCalls = StormMetricsRegistry.registerMeter("nimbus:num-getTopologyInfo-calls");
private static final Meter getTopologyPageInfoCalls = StormMetricsRegistry.registerMeter("nimbus:num-getTopologyPageInfo-calls");
private static final Meter getSupervisorPageInfoCalls = StormMetricsRegistry.registerMeter("nimbus:num-getSupervisorPageInfo-calls");
private static final Meter getComponentPageInfoCalls = StormMetricsRegistry.registerMeter("nimbus:num-getComponentPageInfo-calls");
private static final Histogram scheduleTopologyTimeMs = StormMetricsRegistry.registerHistogram("nimbus:time-scheduleTopology-ms",
- new ExponentiallyDecayingReservoir());
+ new ExponentiallyDecayingReservoir());
private static final Meter getOwnerResourceSummariesCalls = StormMetricsRegistry.registerMeter(
- "nimbus:num-getOwnerResourceSummaries-calls");
+ "nimbus:num-getOwnerResourceSummaries-calls");
+ // END Metrics
private static final Meter shutdownCalls = StormMetricsRegistry.registerMeter("nimbus:num-shutdown-calls");
private static final Meter processWorkerMetricsCalls = StormMetricsRegistry.registerMeter("nimbus:process-worker-metric-calls");
- // END Metrics
-
private static final String STORM_VERSION = VersionInfo.getVersion();
-
- @VisibleForTesting
- public static final List<ACL> ZK_ACLS = Arrays.asList(ZooDefs.Ids.CREATOR_ALL_ACL.get(0),
- new ACL(ZooDefs.Perms.READ | ZooDefs.Perms.CREATE, ZooDefs.Ids.ANYONE_ID_UNSAFE));
-
- public static final SimpleVersion MIN_VERSION_SUPPORT_RPC_HEARTBEAT = new SimpleVersion("2.0.0");
-
- private static List<ACL> getNimbusAcls(Map<String, Object> conf) {
- List<ACL> acls = null;
- if (Utils.isZkAuthenticationConfiguredStormServer(conf)) {
- acls = ZK_ACLS;
- }
- return acls;
- }
-
private static final Subject NIMBUS_SUBJECT = new Subject();
-
- static {
- NIMBUS_SUBJECT.getPrincipals().add(new NimbusPrincipal());
- NIMBUS_SUBJECT.setReadOnly();
- }
-
- // TOPOLOGY STATE TRANSITIONS
- private static StormBase make(TopologyStatus status) {
- StormBase ret = new StormBase();
- ret.set_status(status);
- //The following are required for backwards compatibility with clojure code
- ret.set_component_executors(Collections.emptyMap());
- ret.set_component_debug(Collections.emptyMap());
- return ret;
- }
-
private static final TopologyStateTransition NOOP_TRANSITION = (arg, nimbus, topoId, base) -> null;
private static final TopologyStateTransition INACTIVE_TRANSITION = (arg, nimbus, topoId, base) -> Nimbus.make(TopologyStatus.INACTIVE);
private static final TopologyStateTransition ACTIVE_TRANSITION = (arg, nimbus, topoId, base) -> Nimbus.make(TopologyStatus.ACTIVE);
+ private static final TopologyStateTransition REMOVE_TRANSITION = (args, nimbus, topoId, base) -> {
+ LOG.info("Killing topology: {}", topoId);
+ IStormClusterState state = nimbus.getStormClusterState();
+ Assignment oldAssignment = state.assignmentInfo(topoId, null);
+ state.removeStorm(topoId);
+ notifySupervisorsAsKilled(state, oldAssignment, nimbus.getAssignmentsDistributer());
+ BlobStore store = nimbus.getBlobStore();
+ if (store instanceof LocalFsBlobStore) {
+ for (String key : Nimbus.getKeyListFromId(nimbus.getConf(), topoId)) {
+ state.removeBlobstoreKey(key);
+ state.removeKeyVersion(key);
+ }
+ }
+ nimbus.getHeartbeatsCache().getAndUpdate(new Dissoc<>(topoId));
+ return null;
+ };
+ private static final TopologyStateTransition DO_REBALANCE_TRANSITION = (args, nimbus, topoId, base) -> {
+ nimbus.doRebalance(topoId, base);
+ return Nimbus.make(base.get_prev_status());
+ };
private static final TopologyStateTransition KILL_TRANSITION = (killTime, nimbus, topoId, base) -> {
int delay = 0;
if (killTime != null) {
- delay = ((Number)killTime).intValue();
+ delay = ((Number) killTime).intValue();
} else {
delay = ObjectReader.getInt(Nimbus.readTopoConf(topoId, nimbus.getTopoCache()).get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS));
}
@@ -309,7 +298,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
sb.set_component_debug(Collections.emptyMap());
return sb;
};
-
+
private static final TopologyStateTransition REBALANCE_TRANSITION = (args, nimbus, topoId, base) -> {
RebalanceOptions rbo = ((RebalanceOptions) args).deepCopy();
int delay = 0;
@@ -319,12 +308,12 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
delay = ObjectReader.getInt(Nimbus.readTopoConf(topoId, nimbus.getTopoCache()).get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS));
}
nimbus.delayEvent(topoId, delay, TopologyActions.DO_REBALANCE, null);
-
+
rbo.set_wait_secs(delay);
if (!rbo.is_set_num_executors()) {
rbo.set_num_executors(Collections.emptyMap());
}
-
+
StormBase sb = new StormBase();
sb.set_status(TopologyStatus.REBALANCING);
sb.set_prev_status(base.get_status());
@@ -333,175 +322,228 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
sb.set_topology_action_options(tao);
sb.set_component_executors(Collections.emptyMap());
sb.set_component_debug(Collections.emptyMap());
-
+
return sb;
};
-
private static final TopologyStateTransition STARTUP_WHEN_KILLED_TRANSITION = (args, nimbus, topoId, base) -> {
int delay = base.get_topology_action_options().get_kill_options().get_wait_secs();
nimbus.delayEvent(topoId, delay, TopologyActions.REMOVE, null);
return null;
};
-
- private static final TopologyStateTransition REMOVE_TRANSITION = (args, nimbus, topoId, base) -> {
- LOG.info("Killing topology: {}", topoId);
- IStormClusterState state = nimbus.getStormClusterState();
- Assignment oldAssignment = state.assignmentInfo(topoId, null);
- state.removeStorm(topoId);
- notifySupervisorsAsKilled(state, oldAssignment, nimbus.getAssignmentsDistributer());
- BlobStore store = nimbus.getBlobStore();
- if (store instanceof LocalFsBlobStore) {
- for (String key: Nimbus.getKeyListFromId(nimbus.getConf(), topoId)) {
- state.removeBlobstoreKey(key);
- state.removeKeyVersion(key);
- }
- }
- nimbus.getHeartbeatsCache().getAndUpdate(new Dissoc<>(topoId));
- return null;
- };
-
private static final TopologyStateTransition STARTUP_WHEN_REBALANCING_TRANSITION = (args, nimbus, topoId, base) -> {
int delay = base.get_topology_action_options().get_rebalance_options().get_wait_secs();
nimbus.delayEvent(topoId, delay, TopologyActions.DO_REBALANCE, null);
return null;
};
-
- private static final TopologyStateTransition DO_REBALANCE_TRANSITION = (args, nimbus, topoId, base) -> {
- nimbus.doRebalance(topoId, base);
- return Nimbus.make(base.get_prev_status());
- };
-
private static final Map<TopologyStatus, Map<TopologyActions, TopologyStateTransition>> TOPO_STATE_TRANSITIONS =
- new ImmutableMap.Builder<TopologyStatus, Map<TopologyActions, TopologyStateTransition>>()
+ new ImmutableMap.Builder<TopologyStatus, Map<TopologyActions, TopologyStateTransition>>()
.put(TopologyStatus.ACTIVE, new ImmutableMap.Builder<TopologyActions, TopologyStateTransition>()
- .put(TopologyActions.INACTIVATE, INACTIVE_TRANSITION)
- .put(TopologyActions.ACTIVATE, NOOP_TRANSITION)
- .put(TopologyActions.REBALANCE, REBALANCE_TRANSITION)
- .put(TopologyActions.KILL, KILL_TRANSITION)
- .build())
+ .put(TopologyActions.INACTIVATE, INACTIVE_TRANSITION)
+ .put(TopologyActions.ACTIVATE, NOOP_TRANSITION)
+ .put(TopologyActions.REBALANCE, REBALANCE_TRANSITION)
+ .put(TopologyActions.KILL, KILL_TRANSITION)
+ .build())
.put(TopologyStatus.INACTIVE, new ImmutableMap.Builder<TopologyActions, TopologyStateTransition>()
- .put(TopologyActions.ACTIVATE, ACTIVE_TRANSITION)
- .put(TopologyActions.INACTIVATE, NOOP_TRANSITION)
- .put(TopologyActions.REBALANCE, REBALANCE_TRANSITION)
- .put(TopologyActions.KILL, KILL_TRANSITION)
- .build())
+ .put(TopologyActions.ACTIVATE, ACTIVE_TRANSITION)
+ .put(TopologyActions.INACTIVATE, NOOP_TRANSITION)
+ .put(TopologyActions.REBALANCE, REBALANCE_TRANSITION)
+ .put(TopologyActions.KILL, KILL_TRANSITION)
+ .build())
.put(TopologyStatus.KILLED, new ImmutableMap.Builder<TopologyActions, TopologyStateTransition>()
- .put(TopologyActions.STARTUP, STARTUP_WHEN_KILLED_TRANSITION)
- .put(TopologyActions.KILL, KILL_TRANSITION)
- .put(TopologyActions.REMOVE, REMOVE_TRANSITION)
- .build())
+ .put(TopologyActions.STARTUP, STARTUP_WHEN_KILLED_TRANSITION)
+ .put(TopologyActions.KILL, KILL_TRANSITION)
+ .put(TopologyActions.REMOVE, REMOVE_TRANSITION)
+ .build())
.put(TopologyStatus.REBALANCING, new ImmutableMap.Builder<TopologyActions, TopologyStateTransition>()
- .put(TopologyActions.STARTUP, STARTUP_WHEN_REBALANCING_TRANSITION)
- .put(TopologyActions.KILL, KILL_TRANSITION)
- .put(TopologyActions.DO_REBALANCE, DO_REBALANCE_TRANSITION)
- .build())
+ .put(TopologyActions.STARTUP, STARTUP_WHEN_REBALANCING_TRANSITION)
+ .put(TopologyActions.KILL, KILL_TRANSITION)
+ .put(TopologyActions.DO_REBALANCE, DO_REBALANCE_TRANSITION)
+ .build())
.build();
-
+ private static final List<String> EMPTY_STRING_LIST = Collections.unmodifiableList(Collections.emptyList());
+ private static final Set<String> EMPTY_STRING_SET = Collections.unmodifiableSet(Collections.emptySet());
+ private static final Pattern TOPOLOGY_NAME_REGEX = Pattern.compile("^[^/.:\\\\]+$");
+
// END TOPOLOGY STATE TRANSITIONS
-
- private static final class Assoc<K,V> implements UnaryOperator<Map<K, V>> {
- private final K key;
- private final V value;
-
- public Assoc(K key, V value) {
- this.key = key;
- this.value = value;
- }
-
- @Override
- public Map<K, V> apply(Map<K, V> t) {
- Map<K, V> ret = new HashMap<>(t);
- ret.put(key, value);
- return ret;
- }
+
+ static {
+ NIMBUS_SUBJECT.getPrincipals().add(new NimbusPrincipal());
+ NIMBUS_SUBJECT.setReadOnly();
}
-
- private static final class Dissoc<K,V> implements UnaryOperator<Map<K, V>> {
- private final K key;
-
- public Dissoc(K key) {
- this.key = key;
- }
-
- @Override
- public Map<K, V> apply(Map<K, V> t) {
- Map<K, V> ret = new HashMap<>(t);
- ret.remove(key);
- return ret;
- }
+
+ private final Map<String, Object> conf;
+ private final NavigableMap<SimpleVersion, List<String>> supervisorClasspaths;
+ private final NimbusInfo nimbusHostPortInfo;
+ private final INimbus inimbus;
+ private final IAuthorizer impersonationAuthorizationHandler;
+ private final AtomicLong submittedCount;
+ private final IStormClusterState stormClusterState;
+ private final Object submitLock = new Object();
+ private final Object schedLock = new Object();
+ private final Object credUpdateLock = new Object();
+ private final AtomicReference<Map<String, Map<List<Integer>, Map<String, Object>>>> heartbeatsCache;
+ private final AtomicBoolean heartbeatsReadyFlag;
+ private final IWorkerHeartbeatsRecoveryStrategy heartbeatsRecoveryStrategy;
+ @SuppressWarnings("deprecation")
+ private final TimeCacheMap<String, BufferInputStream> downloaders;
+ @SuppressWarnings("deprecation")
+ private final TimeCacheMap<String, WritableByteChannel> uploaders;
+ private final BlobStore blobStore;
+ private final TopoCache topoCache;
+ @SuppressWarnings("deprecation")
+ private final TimeCacheMap<String, BufferInputStream> blobDownloaders;
+ @SuppressWarnings("deprecation")
+ private final TimeCacheMap<String, OutputStream> blobUploaders;
+ @SuppressWarnings("deprecation")
+ private final TimeCacheMap<String, Iterator<String>> blobListers;
+ private final UptimeComputer uptime;
+ private final ITopologyValidator validator;
+ private final StormTimer timer;
+ private final IScheduler scheduler;
+ private final IScheduler underlyingScheduler;
+ private final ILeaderElector leaderElector;
+ private final AssignmentDistributionService assignmentsDistributer;
+ private final AtomicReference<Map<String, String>> idToSchedStatus;
+ private final AtomicReference<Map<String, SupervisorResources>> nodeIdToResources;
+ private final AtomicReference<Map<String, TopologyResources>> idToResources;
+ private final AtomicReference<Map<String, Map<WorkerSlot, WorkerResources>>> idToWorkerResources;
+ private final Collection<ICredentialsRenewer> credRenewers;
+ private final Object topologyHistoryLock;
+ private final LocalState topologyHistoryState;
+ private final Collection<INimbusCredentialPlugin> nimbusAutocredPlugins;
+ private final ITopologyActionNotifierPlugin nimbusTopologyActionNotifier;
+ private final List<ClusterMetricsConsumerExecutor> clusterConsumerExceutors;
+ private final IGroupMappingServiceProvider groupMapper;
+ private final IPrincipalToLocal principalToLocal;
+ private MetricStore metricsStore;
+ private IAuthorizer authorizationHandler;
+ //Cached CuratorFramework, mainly used for BlobStore.
+ private CuratorFramework zkClient;
+ //May be null if worker tokens are not supported by the thrift transport.
+ private WorkerTokenManager workerTokenManager;
+
+ public Nimbus(Map<String, Object> conf, INimbus inimbus) throws Exception {
+ this(conf, inimbus, null, null, null, null, null);
}
-
- @VisibleForTesting
- public static class StandaloneINimbus implements INimbus {
- @Override
- public void prepare(Map<String, Object> topoConf, String schedulerLocalDir) {
- //NOOP
- }
+ public Nimbus(Map<String, Object> conf, INimbus inimbus, IStormClusterState stormClusterState, NimbusInfo hostPortInfo,
+ BlobStore blobStore, ILeaderElector leaderElector, IGroupMappingServiceProvider groupMapper) throws Exception {
+ this(conf, inimbus, stormClusterState, hostPortInfo, blobStore, null, leaderElector, groupMapper);
+ }
- @SuppressWarnings("unchecked")
- @Override
- public Collection<WorkerSlot> allSlotsAvailableForScheduling(Collection<SupervisorDetails> supervisors,
- Topologies topologies, Set<String> topologiesMissingAssignments) {
- Set<WorkerSlot> ret = new HashSet<>();
- for (SupervisorDetails sd: supervisors) {
- String id = sd.getId();
- for (Number port: (Collection<Number>)sd.getMeta()) {
- ret.add(new WorkerSlot(id, port));
- }
- }
- return ret;
+ public Nimbus(Map<String, Object> conf, INimbus inimbus, IStormClusterState stormClusterState, NimbusInfo hostPortInfo,
+ BlobStore blobStore, TopoCache topoCache, ILeaderElector leaderElector, IGroupMappingServiceProvider groupMapper)
+ throws Exception {
+ this.conf = conf;
+
+ this.metricsStore = null;
+ try {
+ this.metricsStore = MetricStoreConfig.configure(conf);
+ } catch (Exception e) {
+ // the metrics store is not critical to the operation of the cluster, allow Nimbus to come up
+ LOG.error("Failed to initialize metric store", e);
}
- @Override
- public void assignSlots(Topologies topologies, Map<String, Collection<WorkerSlot>> newSlotsByTopologyId) {
- //NOOP
+ if (hostPortInfo == null) {
+ hostPortInfo = NimbusInfo.fromConf(conf);
+ }
+ this.nimbusHostPortInfo = hostPortInfo;
+ if (inimbus != null) {
+ inimbus.prepare(conf, ServerConfigUtils.masterInimbusDir(conf));
}
- @Override
- public String getHostName(Map<String, SupervisorDetails> supervisors, String nodeId) {
- SupervisorDetails sd = supervisors.get(nodeId);
- if (sd != null) {
- return sd.getHost();
- }
- return null;
+ this.inimbus = inimbus;
+ this.authorizationHandler = StormCommon.mkAuthorizationHandler((String) conf.get(DaemonConfig.NIMBUS_AUTHORIZER), conf);
+ this.impersonationAuthorizationHandler =
+ StormCommon.mkAuthorizationHandler((String) conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf);
+ this.submittedCount = new AtomicLong(0);
+ if (stormClusterState == null) {
+ stormClusterState = makeStormClusterState(conf);
+ }
+ this.stormClusterState = stormClusterState;
+ this.heartbeatsCache = new AtomicReference<>(new HashMap<>());
+ this.heartbeatsReadyFlag = new AtomicBoolean(false);
+ this.heartbeatsRecoveryStrategy = WorkerHeartbeatsRecoveryStrategyFactory.getStrategy(conf);
+ this.downloaders = fileCacheMap(conf);
+ this.uploaders = fileCacheMap(conf);
+ if (blobStore == null) {
+ blobStore = ServerUtils.getNimbusBlobStore(conf, this.nimbusHostPortInfo);
+ }
+ this.blobStore = blobStore;
+ if (topoCache == null) {
+ topoCache = new TopoCache(blobStore, conf);
+ }
+ this.topoCache = topoCache;
+ this.blobDownloaders = makeBlobCacheMap(conf);
+ this.blobUploaders = makeBlobCacheMap(conf);
+ this.blobListers = makeBlobListCacheMap(conf);
+ this.uptime = Utils.makeUptimeComputer();
+ this.validator = ReflectionUtils
+ .newInstance((String) conf.getOrDefault(DaemonConfig.NIMBUS_TOPOLOGY_VALIDATOR, DefaultTopologyValidator.class.getName()));
+ this.timer = new StormTimer(null, (t, e) -> {
+ LOG.error("Error while processing event", e);
+ Utils.exitProcess(20, "Error while processing event");
+ });
+ this.underlyingScheduler = makeScheduler(conf, inimbus);
+ this.scheduler = wrapAsBlacklistScheduler(conf, underlyingScheduler);
+ this.zkClient = makeZKClient(conf);
+ if (leaderElector == null) {
+ leaderElector = Zookeeper.zkLeaderElector(conf, zkClient, blobStore, topoCache, stormClusterState, getNimbusAcls(conf));
}
+ this.leaderElector = leaderElector;
+ this.assignmentsDistributer = AssignmentDistributionService.getInstance(conf);
+ this.idToSchedStatus = new AtomicReference<>(new HashMap<>());
+ this.nodeIdToResources = new AtomicReference<>(new HashMap<>());
+ this.idToResources = new AtomicReference<>(new HashMap<>());
+ this.idToWorkerResources = new AtomicReference<>(new HashMap<>());
+ this.credRenewers = AuthUtils.GetCredentialRenewers(conf);
+ this.topologyHistoryLock = new Object();
+ this.topologyHistoryState = ServerConfigUtils.nimbusTopoHistoryState(conf);
+ this.nimbusAutocredPlugins = AuthUtils.getNimbusAutoCredPlugins(conf);
+ this.nimbusTopologyActionNotifier = createTopologyActionNotifier(conf);
+ this.clusterConsumerExceutors = makeClusterMetricsConsumerExecutors(conf);
+ if (groupMapper == null) {
+ groupMapper = AuthUtils.GetGroupMappingServiceProviderPlugin(conf);
+ }
+ this.groupMapper = groupMapper;
+ this.principalToLocal = AuthUtils.GetPrincipalToLocalPlugin(conf);
+ this.supervisorClasspaths = Collections.unmodifiableNavigableMap(
+ Utils.getConfiguredClasspathVersions(conf, EMPTY_STRING_LIST));// We don't use the classpath part of this, so just an empty list
+ }
- @Override
- public IScheduler getForcedScheduler() {
- return null;
+ private static List<ACL> getNimbusAcls(Map<String, Object> conf) {
+ List<ACL> acls = null;
+ if (Utils.isZkAuthenticationConfiguredStormServer(conf)) {
+ acls = ZK_ACLS;
}
-
+ return acls;
}
-
- private static class CommonTopoInfo {
- public Map<String, Object> topoConf;
- public String topoName;
- public StormTopology topology;
- public Map<Integer, String> taskToComponent;
- public StormBase base;
- public int launchTimeSecs;
- public Assignment assignment;
- public Map<List<Integer>, Map<String, Object>> beats;
- public HashSet<String> allComponents;
+ // TOPOLOGY STATE TRANSITIONS
+ private static StormBase make(TopologyStatus status) {
+ StormBase ret = new StormBase();
+ ret.set_status(status);
+ //The following are required for backwards compatibility with clojure code
+ ret.set_component_executors(Collections.emptyMap());
+ ret.set_component_debug(Collections.emptyMap());
+ return ret;
}
-
+
@SuppressWarnings("deprecation")
private static <T extends AutoCloseable> TimeCacheMap<String, T> fileCacheMap(Map<String, Object> conf) {
return new TimeCacheMap<>(ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_FILE_COPY_EXPIRATION_SECS), 600),
- (id, stream) -> {
- try {
- stream.close();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- });
+ (id, stream) -> {
+ try {
+ stream.close();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
}
private static <K, V> Map<K, V> mapDiff(Map<? extends K, ? extends V> first, Map<? extends K, ? extends V> second) {
Map<K, V> ret = new HashMap<>();
- for (Entry<? extends K, ? extends V> entry: second.entrySet()) {
+ for (Entry<? extends K, ? extends V> entry : second.entrySet()) {
if (!entry.getValue().equals(first.get(entry.getKey()))) {
ret.put(entry.getKey(), entry.getValue());
}
@@ -531,30 +573,31 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
/**
- * Constructs a TimeCacheMap instance with a blob store timeout whose
- * expiration callback invokes cancel on the value held by an expired entry when
- * that value is an AtomicOutputStream and calls close otherwise.
+ * Constructs a TimeCacheMap instance with a blob store timeout whose expiration callback invokes cancel on the value held by an expired
+ * entry when that value is an AtomicOutputStream and calls close otherwise.
+ *
* @param conf the config to use
* @return the newly created map
*/
@SuppressWarnings("deprecation")
private static <T extends AutoCloseable> TimeCacheMap<String, T> makeBlobCacheMap(Map<String, Object> conf) {
return new TimeCacheMap<>(ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_BLOBSTORE_EXPIRATION_SECS), 600),
- (id, stream) -> {
- try {
- if (stream instanceof AtomicOutputStream) {
- ((AtomicOutputStream) stream).cancel();
- } else {
- stream.close();
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- });
+ (id, stream) -> {
+ try {
+ if (stream instanceof AtomicOutputStream) {
+ ((AtomicOutputStream) stream).cancel();
+ } else {
+ stream.close();
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
}
-
+
/**
* Constructs a TimeCacheMap instance with a blobstore timeout and no callback function.
+ *
* @param conf the config to use
* @return the newly created TimeCacheMap
*/
@@ -562,7 +605,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
private static TimeCacheMap<String, Iterator<String>> makeBlobListCacheMap(Map<String, Object> conf) {
return new TimeCacheMap<>(ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_BLOBSTORE_EXPIRATION_SECS), 600));
}
-
+
private static ITopologyActionNotifierPlugin createTopologyActionNotifier(Map<String, Object> conf) {
String clazz = (String) conf.get(DaemonConfig.NIMBUS_TOPOLOGY_ACTION_NOTIFIER_PLUGIN);
ITopologyActionNotifierPlugin ret = null;
@@ -577,7 +620,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
return ret;
}
-
+
@SuppressWarnings("unchecked")
private static List<ClusterMetricsConsumerExecutor> makeClusterMetricsConsumerExecutors(Map<String, Object> conf) {
Collection<Map<String, Object>> consumers = (Collection<Map<String, Object>>) conf.get(
@@ -590,16 +633,16 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
return ret;
}
-
+
private static Subject getSubject() {
return ReqContext.context().subject();
}
-
+
static Map<String, Object> readTopoConf(String topoId, TopoCache tc) throws KeyNotFoundException,
AuthorizationException, IOException {
return tc.readTopoConf(topoId, getSubject());
}
-
+
static List<String> getKeyListFromId(Map<String, Object> conf, String id) {
List<String> ret = new ArrayList<>(3);
ret.add(ConfigUtils.masterStormCodeKey(id));
@@ -609,56 +652,57 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
return ret;
}
-
+
private static int getVersionForKey(String key, NimbusInfo nimbusInfo,
- CuratorFramework zkClient) throws KeyNotFoundException {
+ CuratorFramework zkClient) throws KeyNotFoundException {
KeySequenceNumber kseq = new KeySequenceNumber(key, nimbusInfo);
return kseq.getKeySequenceNumber(zkClient);
}
-
+
private static StormTopology readStormTopology(String topoId, TopoCache tc) throws KeyNotFoundException, AuthorizationException,
IOException {
return tc.readTopology(topoId, getSubject());
}
-
+
private static Map<String, Object> readTopoConfAsNimbus(String topoId, TopoCache tc) throws KeyNotFoundException,
AuthorizationException, IOException {
return tc.readTopoConf(topoId, NIMBUS_SUBJECT);
}
-
+
private static StormTopology readStormTopologyAsNimbus(String topoId, TopoCache tc) throws KeyNotFoundException,
AuthorizationException, IOException {
return tc.readTopology(topoId, NIMBUS_SUBJECT);
}
-
+
/**
- * convert {topology-id -> SchedulerAssignment} to
- * {topology-id -> {executor [node port]}}.
+ * convert {topology-id -> SchedulerAssignment} to {topology-id -> {executor [node port]}}.
+ *
* @return {topology-id -> {executor [node port]}} mapping
*/
- private static Map<String, Map<List<Long>, List<Object>>> computeTopoToExecToNodePort(Map<String, SchedulerAssignment> schedAssignments) {
+ private static Map<String, Map<List<Long>, List<Object>>> computeTopoToExecToNodePort(
+ Map<String, SchedulerAssignment> schedAssignments) {
Map<String, Map<List<Long>, List<Object>>> ret = new HashMap<>();
- for (Entry<String, SchedulerAssignment> schedEntry: schedAssignments.entrySet()) {
+ for (Entry<String, SchedulerAssignment> schedEntry : schedAssignments.entrySet()) {
Map<List<Long>, List<Object>> execToNodePort = new HashMap<>();
- for (Entry<ExecutorDetails, WorkerSlot> execAndNodePort: schedEntry.getValue().getExecutorToSlot().entrySet()) {
+ for (Entry<ExecutorDetails, WorkerSlot> execAndNodePort : schedEntry.getValue().getExecutorToSlot().entrySet()) {
ExecutorDetails exec = execAndNodePort.getKey();
WorkerSlot slot = execAndNodePort.getValue();
-
+
List<Long> listExec = new ArrayList<>(2);
listExec.add((long) exec.getStartTask());
listExec.add((long) exec.getEndTask());
-
+
List<Object> nodePort = new ArrayList<>(2);
nodePort.add(slot.getNodeId());
- nodePort.add((long)slot.getPort());
-
+ nodePort.add((long) slot.getPort());
+
execToNodePort.put(listExec, nodePort);
}
ret.put(schedEntry.getKey(), execToNodePort);
}
return ret;
}
-
+
private static int numUsedWorkers(SchedulerAssignment assignment) {
if (assignment == null) {
return 0;
@@ -667,8 +711,8 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
/**
- * Convert {topology-id -> SchedulerAssignment} to {topology-id -> {WorkerSlot WorkerResources}}. Make sure this
- * can deal with other non-RAS schedulers later we may further support map-for-any-resources.
+ * Convert {topology-id -> SchedulerAssignment} to {topology-id -> {WorkerSlot WorkerResources}}. Make sure this can deal with other
+ * non-RAS schedulers later we may further support map-for-any-resources.
*
* @param schedAssignments the assignments
* @return The resources used per slot
@@ -687,7 +731,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
Map<String, Map<List<Long>, List<Object>>> ret = computeTopoToExecToNodePort(schedAssignments);
// Print some useful information
if (existingAssignments != null && !existingAssignments.isEmpty()) {
- for (Entry<String, Map<List<Long>, List<Object>>> entry: ret.entrySet()) {
+ for (Entry<String, Map<List<Long>, List<Object>>> entry : ret.entrySet()) {
String topoId = entry.getKey();
Map<List<Long>, List<Object>> execToNodePort = entry.getValue();
Assignment assignment = existingAssignments.get(topoId);
@@ -696,12 +740,12 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
Map<List<Long>, NodeInfo> old = assignment.get_executor_node_port();
Map<List<Long>, List<Object>> reassigned = new HashMap<>();
- for (Entry<List<Long>, List<Object>> execAndNodePort: execToNodePort.entrySet()) {
+ for (Entry<List<Long>, List<Object>> execAndNodePort : execToNodePort.entrySet()) {
NodeInfo oldAssigned = old.get(execAndNodePort.getKey());
String node = (String) execAndNodePort.getValue().get(0);
Long port = (Long) execAndNodePort.getValue().get(1);
- if (oldAssigned == null || !oldAssigned.get_node().equals(node)
- || !port.equals(oldAssigned.get_port_iterator().next())) {
+ if (oldAssigned == null || !oldAssigned.get_node().equals(node)
+ || !port.equals(oldAssigned.get_port_iterator().next())) {
reassigned.put(execAndNodePort.getKey(), execAndNodePort.getValue());
}
}
@@ -716,12 +760,12 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
return ret;
}
-
+
private static List<List<Long>> changedExecutors(Map<List<Long>, NodeInfo> map, Map<List<Long>,
List<Object>> newExecToNodePort) {
HashMap<NodeInfo, List<List<Long>>> tmpSlotAssigned = map == null ? new HashMap<>() : Utils.reverseMap(map);
HashMap<List<Object>, List<List<Long>>> slotAssigned = new HashMap<>();
- for (Entry<NodeInfo, List<List<Long>>> entry: tmpSlotAssigned.entrySet()) {
+ for (Entry<NodeInfo, List<List<Long>>> entry : tmpSlotAssigned.entrySet()) {
NodeInfo ni = entry.getKey();
List<Object> key = new ArrayList<>(2);
key.add(ni.get_node());
@@ -731,16 +775,16 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
slotAssigned.put(key, value);
}
HashMap<List<Object>, List<List<Long>>> tmpNewSlotAssigned = newExecToNodePort == null ? new HashMap<>() :
- Utils.reverseMap(newExecToNodePort);
+ Utils.reverseMap(newExecToNodePort);
HashMap<List<Object>, List<List<Long>>> newSlotAssigned = new HashMap<>();
- for (Entry<List<Object>, List<List<Long>>> entry: tmpNewSlotAssigned.entrySet()) {
+ for (Entry<List<Object>, List<List<Long>>> entry : tmpNewSlotAssigned.entrySet()) {
List<List<Long>> value = new ArrayList<>(entry.getValue());
value.sort((a, b) -> a.get(0).compareTo(b.get(0)));
newSlotAssigned.put(entry.getKey(), value);
}
Map<List<Object>, List<List<Long>>> diff = mapDiff(slotAssigned, newSlotAssigned);
List<List<Long>> ret = new ArrayList<>();
- for (List<List<Long>> val: diff.values()) {
+ for (List<List<Long>> val : diff.values()) {
ret.addAll(val);
}
return ret;
@@ -751,27 +795,27 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
Set<NodeInfo> niRet = new HashSet<>(current.get_executor_node_port().values());
niRet.removeAll(oldSlots);
Set<WorkerSlot> ret = new HashSet<>();
- for (NodeInfo ni: niRet) {
+ for (NodeInfo ni : niRet) {
ret.add(new WorkerSlot(ni.get_node(), ni.get_port_iterator().next()));
}
return ret;
}
-
+
private static Map<String, SupervisorDetails> basicSupervisorDetailsMap(IStormClusterState state) {
Map<String, SupervisorDetails> ret = new HashMap<>();
- for (Entry<String, SupervisorInfo> entry: state.allSupervisorInfo().entrySet()) {
+ for (Entry<String, SupervisorInfo> entry : state.allSupervisorInfo().entrySet()) {
String id = entry.getKey();
SupervisorInfo info = entry.getValue();
ret.put(id, new SupervisorDetails(id, info.get_server_port(), info.get_hostname(),
- info.get_scheduler_meta(), null, info.get_resources_map()));
+ info.get_scheduler_meta(), null, info.get_resources_map()));
}
return ret;
}
-
+
private static boolean isTopologyActive(IStormClusterState state, String topoName) {
return state.getTopoId(topoName).isPresent();
}
-
+
private static Map<String, Object> tryReadTopoConf(String topoId, TopoCache tc)
throws NotAliveException, AuthorizationException, IOException {
try {
@@ -785,10 +829,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
throw new NotAliveException(topoId);
}
}
-
- private static final List<String> EMPTY_STRING_LIST = Collections.unmodifiableList(Collections.emptyList());
- private static final Set<String> EMPTY_STRING_SET = Collections.unmodifiableSet(Collections.emptySet());
-
+
@VisibleForTesting
public static Set<String> topoIdsToClean(IStormClusterState state, BlobStore store) {
Set<String> ret = new HashSet<>();
@@ -800,7 +841,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
ret.removeAll(Utils.OR(state.activeStorms(), EMPTY_STRING_LIST));
return ret;
}
-
+
private static String extractStatusStr(StormBase base) {
String ret = null;
if (base != null) {
@@ -815,7 +856,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
private static StormTopology normalizeTopology(Map<String, Object> topoConf, StormTopology topology)
throws InvalidTopologyException {
StormTopology ret = topology.deepCopy();
- for (Object comp: StormCommon.allComponents(ret).values()) {
+ for (Object comp : StormCommon.allComponents(ret).values()) {
Map<String, Object> mergedConf = StormCommon.componentConf(comp);
mergedConf.put(Config.TOPOLOGY_TASKS, ServerUtils.getComponentParallelism(topoConf, comp));
String jsonConf = JSONValue.toJSONString(mergedConf);
@@ -823,47 +864,47 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
return ret;
}
-
+
private static void addToDecorators(Set<String> decorators, List<String> conf) {
if (conf != null) {
decorators.addAll(conf);
}
}
-
+
@SuppressWarnings("unchecked")
private static void addToSerializers(Map<String, String> ser, List<Object> conf) {
if (conf != null) {
- for (Object o: conf) {
+ for (Object o : conf) {
if (o instanceof Map) {
- ser.putAll((Map<String,String>)o);
+ ser.putAll((Map<String, String>) o);
} else {
- ser.put((String)o, null);
+ ser.put((String) o, null);
}
}
}
}
-
+
@SuppressWarnings("unchecked")
- private static Map<String, Object> normalizeConf(Map<String,Object> conf, Map<String, Object> topoConf, StormTopology topology) {
+ private static Map<String, Object> normalizeConf(Map<String, Object> conf, Map<String, Object> topoConf, StormTopology topology) {
//ensure that serializations are same for all tasks no matter what's on
// the supervisors. this also allows you to declare the serializations as a sequence
List<Map<String, Object>> allConfs = new ArrayList<>();
- for (Object comp: StormCommon.allComponents(topology).values()) {
+ for (Object comp : StormCommon.allComponents(topology).values()) {
allConfs.add(StormCommon.componentConf(comp));
}
Set<String> decorators = new HashSet<>();
//Yes we are putting in a config that is not the same type we pulled out.
Map<String, String> serializers = new HashMap<>();
- for (Map<String, Object> c: allConfs) {
+ for (Map<String, Object> c : allConfs) {
addToDecorators(decorators, (List<String>) c.get(Config.TOPOLOGY_KRYO_DECORATORS));
addToSerializers(serializers, (List<Object>) c.get(Config.TOPOLOGY_KRYO_REGISTER));
}
- addToDecorators(decorators, (List<String>)topoConf.getOrDefault(Config.TOPOLOGY_KRYO_DECORATORS,
- conf.get(Config.TOPOLOGY_KRYO_DECORATORS)));
- addToSerializers(serializers, (List<Object>)topoConf.getOrDefault(Config.TOPOLOGY_KRYO_REGISTER,
- conf.get(Config.TOPOLOGY_KRYO_REGISTER)));
-
+ addToDecorators(decorators, (List<String>) topoConf.getOrDefault(Config.TOPOLOGY_KRYO_DECORATORS,
+ conf.get(Config.TOPOLOGY_KRYO_DECORATORS)));
+ addToSerializers(serializers, (List<Object>) topoConf.getOrDefault(Config.TOPOLOGY_KRYO_REGISTER,
+ conf.get(Config.TOPOLOGY_KRYO_REGISTER)));
+
Map<String, Object> mergedConf = Utils.merge(conf, topoConf);
Map<String, Object> ret = new HashMap<>(topoConf);
ret.put(Config.TOPOLOGY_KRYO_REGISTER, serializers);
@@ -873,7 +914,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
ret.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, mergedConf.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM));
return ret;
}
-
+
private static void rmBlobKey(BlobStore store, String key, IStormClusterState state) {
try {
store.deleteBlob(key, NIMBUS_SUBJECT);
@@ -885,10 +926,11 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
LOG.info("Exception {}", e);
}
}
-
+
/**
* Deletes jar files in dirLoc older than seconds.
- * @param dirLoc the location to look in for file
+ *
+ * @param dirLoc the location to look in for file
* @param seconds how old is too old and should be deleted
*/
@VisibleForTesting
@@ -904,12 +946,10 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
}
}
-
+
private static ExecutorInfo toExecInfo(List<Long> exec) {
return new ExecutorInfo(exec.get(0).intValue(), exec.get(1).intValue());
}
-
- private static final Pattern TOPOLOGY_NAME_REGEX = Pattern.compile("^[^/.:\\\\]+$");
private static void validateTopologyName(String name) throws InvalidTopologyException {
Matcher m = TOPOLOGY_NAME_REGEX.matcher(name);
@@ -919,16 +959,16 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
private static StormTopology tryReadTopology(String topoId, TopoCache tc)
- throws NotAliveException, AuthorizationException, IOException {
+ throws NotAliveException, AuthorizationException, IOException {
try {
return readStormTopologyAsNimbus(topoId, tc);
} catch (KeyNotFoundException e) {
throw new NotAliveException(topoId);
}
}
-
+
private static void validateTopologySize(Map<String, Object> topoConf, Map<String, Object> nimbusConf,
- StormTopology topology) throws InvalidTopologyException {
+ StormTopology topology) throws InvalidTopologyException {
int workerCount = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_WORKERS), 1);
Integer allowedWorkers = ObjectReader.getInt(nimbusConf.get(DaemonConfig.NIMBUS_SLOTS_PER_TOPOLOGY), null);
int executorsCount = 0;
@@ -938,15 +978,15 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
Integer allowedExecutors = ObjectReader.getInt(nimbusConf.get(DaemonConfig.NIMBUS_EXECUTORS_PER_TOPOLOGY), null);
if (allowedExecutors != null && executorsCount > allowedExecutors) {
throw new InvalidTopologyException("Failed to submit topology. Topology requests more than " +
- allowedExecutors + " executors.");
+ allowedExecutors + " executors.");
}
-
+
if (allowedWorkers != null && workerCount > allowedWorkers) {
throw new InvalidTopologyException("Failed to submit topology. Topology requests more than " +
- allowedWorkers + " workers.");
+ allowedWorkers + " workers.");
}
}
-
+
private static void setLoggerTimeouts(LogLevel level) {
int timeoutSecs = level.get_reset_log_level_timeout_secs();
if (timeoutSecs > 0) {
@@ -955,45 +995,45 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
level.unset_reset_log_level_timeout_epoch();
}
}
-
+
@VisibleForTesting
public static List<String> topologiesOnSupervisor(Map<String, Assignment> assignments, String supervisorId) {
Set<String> ret = new HashSet<>();
- for (Entry<String, Assignment> entry: assignments.entrySet()) {
+ for (Entry<String, Assignment> entry : assignments.entrySet()) {
Assignment assignment = entry.getValue();
- for (NodeInfo nodeInfo: assignment.get_executor_node_port().values()) {
+ for (NodeInfo nodeInfo : assignment.get_executor_node_port().values()) {
if (supervisorId.equals(nodeInfo.get_node())) {
ret.add(entry.getKey());
break;
}
}
}
-
+
return new ArrayList<>(ret);
}
-
+
private static IClusterMetricsConsumer.ClusterInfo mkClusterInfo() {
return new IClusterMetricsConsumer.ClusterInfo(Time.currentTimeSecs());
}
-
+
private static List<DataPoint> extractClusterMetrics(ClusterSummary summ) {
List<DataPoint> ret = new ArrayList<>();
ret.add(new DataPoint("supervisors", summ.get_supervisors_size()));
ret.add(new DataPoint("topologies", summ.get_topologies_size()));
-
+
int totalSlots = 0;
int usedSlots = 0;
- for (SupervisorSummary sup: summ.get_supervisors()) {
+ for (SupervisorSummary sup : summ.get_supervisors()) {
usedSlots += sup.get_num_used_workers();
totalSlots += sup.get_num_workers();
}
ret.add(new DataPoint("slotsTotal", totalSlots));
ret.add(new DataPoint("slotsUsed", usedSlots));
ret.add(new DataPoint("slotsFree", totalSlots - usedSlots));
-
+
int totalExecutors = 0;
int totalTasks = 0;
- for (TopologySummary topo: summ.get_topologies()) {
+ for (TopologySummary topo : summ.get_topologies()) {
totalExecutors += topo.get_num_executors();
totalTasks += topo.get_num_tasks();
}
@@ -1004,8 +1044,9 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
private static Map<IClusterMetricsConsumer.SupervisorInfo, List<DataPoint>> extractSupervisorMetrics(ClusterSummary summ) {
Map<IClusterMetricsConsumer.SupervisorInfo, List<DataPoint>> ret = new HashMap<>();
- for (SupervisorSummary sup: summ.get_supervisors()) {
- IClusterMetricsConsumer.SupervisorInfo info = new IClusterMetricsConsumer.SupervisorInfo(sup.get_host(), sup.get_supervisor_id(), Time.currentTimeSecs());
+ for (SupervisorSummary sup : summ.get_supervisors()) {
+ IClusterMetricsConsumer.SupervisorInfo info =
+ new IClusterMetricsConsumer.SupervisorInfo(sup.get_host(), sup.get_supervisor_id(), Time.currentTimeSecs());
List<DataPoint> metrics = new ArrayList<>();
metrics.add(new DataPoint("slotsTotal", sup.get_num_workers()));
metrics.add(new DataPoint("slotsUsed", sup.get_num_used_workers()));
@@ -1017,7 +1058,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
return ret;
}
-
+
private static void setResourcesDefaultIfNotSet(Map<String, NormalizedResourceRequest> compResourcesMap, String compId,
Map<String, Object> topoConf) {
NormalizedResourceRequest resources = compResourcesMap.get(compId);
@@ -1025,7 +1066,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
compResourcesMap.put(compId, new NormalizedResourceRequest(topoConf));
}
}
-
+
private static void validatePortAvailable(Map<String, Object> conf) throws IOException {
int port = ObjectReader.getInt(conf.get(Config.NIMBUS_THRIFT_PORT));
try (ServerSocket socket = new ServerSocket(port)) {
@@ -1035,7 +1076,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
System.exit(0);
}
}
-
+
private static Nimbus launchServer(Map<String, Object> conf, INimbus inimbus) throws Exception {
StormCommon.validateDistributedMode(conf);
validatePortAvailable(conf);
@@ -1056,7 +1097,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
public static Nimbus launch(INimbus inimbus) throws Exception {
Map<String, Object> conf = Utils.merge(Utils.readStormConfig(),
- ConfigUtils.readYamlConfig("storm-cluster-auth.yaml", false));
+ ConfigUtils.readYamlConfig("storm-cluster-auth.yaml", false));
boolean fixupAcl = (boolean) conf.get(DaemonConfig.STORM_NIMBUS_ZOOKEEPER_ACLS_FIXUP);
boolean checkAcl = fixupAcl || (boolean) conf.get(DaemonConfig.STORM_NIMBUS_ZOOKEEPER_ACLS_CHECK);
if (checkAcl) {
@@ -1064,168 +1105,165 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
return launchServer(conf, inimbus);
}
-
+
public static void main(String[] args) throws Exception {
Utils.setupDefaultUncaughtExceptionHandler();
launch(new StandaloneINimbus());
}
-
- private final Map<String, Object> conf;
- private MetricStore metricsStore;
- private final NavigableMap<SimpleVersion, List<String>> supervisorClasspaths;
- private final NimbusInfo nimbusHostPortInfo;
- private final INimbus inimbus;
- private IAuthorizer authorizationHandler;
- private final IAuthorizer impersonationAuthorizationHandler;
- private final AtomicLong submittedCount;
- //Cached CuratorFramework, mainly used for BlobStore.
- private CuratorFramework zkClient;
- private final IStormClusterState stormClusterState;
- private final Object submitLock = new Object();
- private final Object schedLock = new Object();
- private final Object credUpdateLock = new Object();
- private final AtomicReference<Map<String, Map<List<Integer>, Map<String, Object>>>> heartbeatsCache;
- private final AtomicBoolean heartbeatsReadyFlag;
- private final IWorkerHeartbeatsRecoveryStrategy heartbeatsRecoveryStrategy;
- @SuppressWarnings("deprecation")
- private final TimeCacheMap<String, BufferInputStream> downloaders;
- @SuppressWarnings("deprecation")
- private final TimeCacheMap<String, WritableByteChannel> uploaders;
- private final BlobStore blobStore;
- private final TopoCache topoCache;
- @SuppressWarnings("deprecation")
- private final TimeCacheMap<String, BufferInputStream> blobDownloaders;
- @SuppressWarnings("deprecation")
- private final TimeCacheMap<String, OutputStream> blobUploaders;
- @SuppressWarnings("deprecation")
- private final TimeCacheMap<String, Iterator<String>> blobListers;
- private final UptimeComputer uptime;
- private final ITopologyValidator validator;
- private final StormTimer timer;
- private final IScheduler scheduler;
- private final IScheduler underlyingScheduler;
- private final ILeaderElector leaderElector;
- private final AssignmentDistributionService assignmentsDistributer;
- private final AtomicReference<Map<String, String>> idToSchedStatus;
- private final AtomicReference<Map<String, SupervisorResources>> nodeIdToResources;
- private final AtomicReference<Map<String, TopologyResources>> idToResources;
- private final AtomicReference<Map<String, Map<WorkerSlot, WorkerResources>>> idToWorkerResources;
- private final Collection<ICredentialsRenewer> credRenewers;
- private final Object topologyHistoryLock;
- private final LocalState topologyHistoryState;
- private final Collection<INimbusCredentialPlugin> nimbusAutocredPlugins;
- private final ITopologyActionNotifierPlugin nimbusTopologyActionNotifier;
- private final List<ClusterMetricsConsumerExecutor> clusterConsumerExceutors;
- private final IGroupMappingServiceProvider groupMapper;
- private final IPrincipalToLocal principalToLocal;
- //May be null if worker tokens are not supported by the thrift transport.
- private WorkerTokenManager workerTokenManager;
- private static CuratorFramework makeZKClient(Map<String, Object> conf) {
- List<String> servers = (List<String>)conf.get(Config.STORM_ZOOKEEPER_SERVERS);
- Object port = conf.get(Config.STORM_ZOOKEEPER_PORT);
- String root = (String)conf.get(Config.STORM_ZOOKEEPER_ROOT);
- CuratorFramework ret = null;
- if (servers != null && port != null) {
- ret = ClientZookeeper.mkClient(conf, servers, port, root, new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS);
+ private static CuratorFramework makeZKClient(Map<String, Object> conf) {
+ List<String> servers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS);
+ Object port = conf.get(Config.STORM_ZOOKEEPER_PORT);
+ String root = (String) conf.get(Config.STORM_ZOOKEEPER_ROOT);
+ CuratorFramework ret = null;
+ if (servers != null && port != null) {
+ ret = ClientZookeeper.mkClient(conf, servers, port, root, new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS);
+ }
+ return ret;
+ }
+
+ private static IStormClusterState makeStormClusterState(Map<String, Object> conf) throws Exception {
+ return ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf));
+ }
+
+ private static List<Integer> asIntExec(List<Long> exec) {
+ List<Integer> ret = new ArrayList<>(2);
+ ret.add(exec.get(0).intValue());
+ ret.add(exec.get(1).intValue());
+ return ret;
+ }
+
+ /**
+ * Diff old/new assignment to find nodes which assigned assignments has changed.
+ *
+ * @param oldAss old assigned assignment
+ * @param newAss new assigned assignment
+ * @return nodeId -> host map of assignments changed nodes
+ */
+ private static Map<String, String> assignmentChangedNodes(Assignment oldAss, Assignment newAss) {
+ Map<List<Long>, NodeInfo> oldExecutorNodePort = null;
+ Map<List<Long>, NodeInfo> newExecutorNodePort = null;
+ Map<String, String> allNodeHost = new HashMap<>();
+ if (oldAss != null) {
+ oldExecutorNodePort = oldAss.get_executor_node_port();
+ allNodeHost.putAll(oldAss.get_node_host());
+ }
+ if (newAss != null) {
+ newExecutorNodePort = newAss.get_executor_node_port();
+ allNodeHost.putAll(newAss.get_node_host());
+ }
+ //kill or newly submit
+ if (oldAss == null || newAss == null) {
+ return allNodeHost;
+ } else {
+ // rebalance
+ Map<String, String> ret = new HashMap();
+ for (Map.Entry<List<Long>, NodeInfo> entry : newExecutorNodePort.entrySet()) {
+ NodeInfo newNodeInfo = entry.getValue();
+ NodeInfo oldNodeInfo = oldExecutorNodePort.get(entry.getKey());
+ if (null != oldNodeInfo) {
+ if (!oldNodeInfo.equals(newNodeInfo)) {
+ ret.put(oldNodeInfo.get_node(), allNodeHost.get(oldNodeInfo.get_node()));
+ ret.put(newNodeInfo.get_node(), allNodeHost.get(newNodeInfo.get_node()));
+ }
+ } else {
+ ret.put(newNodeInfo.get_node(), allNodeHost.get(newNodeInfo.get_node()));
+ }
+ }
+
+ return ret;
}
- return ret;
- }
-
- private static IStormClusterState makeStormClusterState(Map<String, Object> conf) throws Exception {
- return ClusterUtils.mkStormClusterState(conf, new ClusterStateContext(DaemonType.NIMBUS, conf));
}
-
- public Nimbus(Map<String, Object> conf, INimbus inimbus) throws Exception {
- this(conf, inimbus, null, null, null, null, null);
+
+ /**
+ * Pick out assignments for specific node from all assignments.
+ *
+ * @param assignmentMap stormId -> assignment map
+ * @param nodeId supervisor/node id
+ * @return stormId -> assignment map for the node
+ */
+ private static Map<String, Assignment> assignmentsForNode(Map<String, Assignment> assignmentMap, String nodeId) {
+ Map<String, Assignment> ret = new HashMap<>();
+ assignmentMap.entrySet().stream().filter(assignmentEntry -> assignmentEntry.getValue().get_node_host().keySet()
+ .contains(nodeId))
+ .forEach(assignmentEntry -> { ret.put(assignmentEntry.getKey(), assignmentEntry.getValue()); });
+
+ return ret;
}
-
- public Nimbus(Map<String, Object> conf, INimbus inimbus, IStormClusterState stormClusterState, NimbusInfo hostPortInfo,
- BlobStore blobStore, ILeaderElector leaderElector, IGroupMappingServiceProvider groupMapper) throws Exception {
- this(conf, inimbus, stormClusterState, hostPortInfo, blobStore, null, leaderElector, groupMapper);
+
+ /**
+ * Notify supervisors/nodes assigned assignments.
+ *
+ * @param assignments assignments map for nodes
+ * @param service {@link AssignmentDistributionService} for distributing assignments asynchronous
+ * @param nodeHost node -> host map
+ * @param supervisorDetails nodeId -> {@link SupervisorDetails} map
+ */
+ private static void notifySupervisorsAssignments(Map<String, Assignment> assignments,
+ AssignmentDistributionService service, Map<String, String> nodeHost,
+ Map<String, SupervisorDetails> supervisorDetails) {
+ for (Map.Entry<String, String> nodeEntry : nodeHost.entrySet()) {
+ try {
+ String nodeId = nodeEntry.getKey();
+ SupervisorAssignments supervisorAssignments = new SupervisorAssignments();
+ supervisorAssignments.set_storm_assignment(assignmentsForNode(assignments, nodeEntry.getKey()));
+ SupervisorDetails details = supervisorDetails.get(nodeId);
+ Integer serverPort = details != null ? details.getServerPort() : null;
+ service.addAssignmentsForNode(nodeId, nodeEntry.getValue(), serverPort, supervisorAssignments);
+ } catch (Throwable tr1) {
+ //just skip when any error happens wait for next round assignments reassign
+ LOG.error("Exception when add assignments distribution task for node {}", nodeEntry.getKey());
+ }
+ }
}
- public Nimbus(Map<String, Object> conf, INimbus inimbus, IStormClusterState stormClusterState, NimbusInfo hostPortInfo,
- BlobStore blobStore, TopoCache topoCache, ILeaderElector leaderElector, IGroupMappingServiceProvider groupMapper)
- throws Exception {
- this.conf = conf;
+ private static void notifySupervisorsAsKilled(IStormClusterState clusterState, Assignment oldAss,
+ AssignmentDistributionService service) {
+ Map<String, String> nodeHost = assignmentChangedNodes(oldAss, null);
+ notifySupervisorsAssignments(clusterState.assignmentsInfo(), service, nodeHost,
+ basicSupervisorDetailsMap(clusterState));
+ }
- this.metricsStore = null;
- try {
- this.metricsStore = MetricStoreConfig.configure(conf);
- } catch (Exception e) {
- // the metrics store is not critical to the operation of the cluster, allow Nimbus to come up
- LOG.error("Failed to initialize metric store", e);
+ @VisibleForTesting
+ static void validateTopologyWorkerMaxHeapSizeConfigs(
+ Map<String, Object> stormConf, StormTopology topology, double defaultWorkerMaxHeapSizeMB) {
+ double largestMemReq = getMaxExecutorMemoryUsageForTopo(topology, stormConf);
+ double topologyWorkerMaxHeapSize =
+ ObjectReader.getDouble(stormConf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB), defaultWorkerMaxHeapSizeMB);
+ if (topologyWorkerMaxHeapSize < largestMemReq) {
+ throw new IllegalArgumentException(
+ "Topology will not be able to be successfully scheduled: Config "
+ + "TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB="
+ + topologyWorkerMaxHeapSize
+ + " < " + largestMemReq + " (Largest memory requirement of a component in the topology)."
+ + " Perhaps set TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB to a larger amount");
}
+ }
- if (hostPortInfo == null) {
- hostPortInfo = NimbusInfo.fromConf(conf);
- }
- this.nimbusHostPortInfo = hostPortInfo;
- if (inimbus != null) {
- inimbus.prepare(conf, ServerConfigUtils.masterInimbusDir(conf));
- }
-
- this.inimbus = inimbus;
- this.authorizationHandler = StormCommon.mkAuthorizationHandler((String) conf.get(DaemonConfig.NIMBUS_AUTHORIZER), conf);
- this.impersonationAuthorizationHandler = StormCommon.mkAuthorizationHandler((String) conf.get(DaemonConfig.NIMBUS_IMPERSONATION_AUTHORIZER), conf);
- this.submittedCount = new AtomicLong(0);
- if (stormClusterState == null) {
- stormClusterState = makeStormClusterState(conf);
- }
- this.stormClusterState = stormClusterState;
- this.heartbeatsCache = new AtomicReference<>(new HashMap<>());
- this.heartbeatsReadyFlag = new AtomicBoolean(false);
- this.heartbeatsRecoveryStrategy = WorkerHeartbeatsRecoveryStrategyFactory.getStrategy(conf);
- this.downloaders = fileCacheMap(conf);
- this.uploaders = fileCacheMap(conf);
- if (blobStore == null) {
- blobStore = ServerUtils.getNimbusBlobStore(conf, this.nimbusHostPortInfo);
- }
- this.blobStore = blobStore;
- if (topoCache == null) {
- topoCache = new TopoCache(blobStore, conf);
- }
- this.topoCache = topoCache;
- this.blobDownloaders = makeBlobCacheMap(conf);
- this.blobUploaders = makeBlobCacheMap(conf);
- this.blobListers = makeBlobListCacheMap(conf);
- this.uptime = Utils.makeUptimeComputer();
- this.validator = ReflectionUtils.newInstance((String) conf.getOrDefault(DaemonConfig.NIMBUS_TOPOLOGY_VALIDATOR, DefaultTopologyValidator.class.getName()));
- this.timer = new StormTimer(null, (t, e) -> {
- LOG.error("Error while processing event", e);
- Utils.exitProcess(20, "Error while processing event");
- });
- this.underlyingScheduler = makeScheduler(conf, inimbus);
- this.scheduler = wrapAsBlacklistScheduler(conf, underlyingScheduler);
- this.zkClient = makeZKClient(conf);
- if (leaderElector == null) {
- leaderElector = Zookeeper.zkLeaderElector(conf, zkClient, blobStore, topoCache, stormClusterState, getNimbusAcls(conf));
+ private static double getMaxExecutorMemoryUsageForTopo(
+ StormTopology topology, Map<String, Object> topologyConf) {
+ double largestMemoryOperator = 0.0;
+ for (NormalizedResourceRequest entry :
+ ResourceUtils.getBoltsResources(topology, topologyConf).values()) {
+ double memoryRequirement = entry.getTotalMemoryMb();
+ if (memoryRequirement > largestMemoryOperator) {
+ largestMemoryOperator = memoryRequirement;
+ }
}
- this.leaderElector = leaderElector;
- this.assignmentsDistributer = AssignmentDistributionService.getInstance(conf);
- this.idToSchedStatus = new AtomicReference<>(new HashMap<>());
- this.nodeIdToResources = new AtomicReference<>(new HashMap<>());
- this.idToResources = new AtomicReference<>(new HashMap<>());
- this.idToWorkerResources = new AtomicReference<>(new HashMap<>());
- this.credRenewers = AuthUtils.GetCredentialRenewers(conf);
- this.topologyHistoryLock = new Object();
- this.topologyHistoryState = ServerConfigUtils.nimbusTopoHistoryState(conf);
- this.nimbusAutocredPlugins = AuthUtils.getNimbusAutoCredPlugins(conf);
- this.nimbusTopologyActionNotifier = createTopologyActionNotifier(conf);
- this.clusterConsumerExceutors = makeClusterMetricsConsumerExecutors(conf);
- if (groupMapper == null) {
- groupMapper = AuthUtils.GetGroupMappingServiceProviderPlugin(conf);
+ for (NormalizedResourceRequest entry :
+ ResourceUtils.getSpoutsResources(topology, topologyConf).values()) {
+ double memoryRequirement = entry.getTotalMemoryMb();
+ if (memoryRequirement > largestMemoryOperator) {
+ largestMemoryOperator = memoryRequirement;
+ }
}
- this.groupMapper = groupMapper;
- this.principalToLocal = AuthUtils.GetPrincipalToLocalPlugin(conf);
- this.supervisorClasspaths = Collections.unmodifiableNavigableMap(Utils.getConfiguredClasspathVersions(conf, EMPTY_STRING_LIST));// We don't use the classpath part of this, so just an empty list
+ return largestMemoryOperator;
}
Map<String, Object> getConf() {
return conf;
}
-
+
@VisibleForTesting
public void setAuthorizationHandler(IAuthorizer authorizationHandler) {
this.authorizationHandler = authorizationHandler;
@@ -1238,9 +1276,9 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
private AssignmentDistributionService getAssignmentsDistributer() {
return assignmentsDistributer;
}
-
+
@VisibleForTesting
- public AtomicReference<Map<String,Map<List<Integer>,Map<String,Object>>>> getHeartbeatsCache() {
+ public AtomicReference<Map<String, Map<List<Integer>, Map<String, Object>>>> getHeartbeatsCache() {
return heartbeatsCache;
}
@@ -1262,26 +1300,27 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
private boolean isLeader() throws Exception {
return leaderElector.isLeader();
}
-
+
private void assertIsLeader() throws Exception {
if (!isLeader()) {
NimbusInfo leaderAddress = leaderElector.getLeader();
throw new RuntimeException("not a leader, current leader is " + leaderAddress);
}
}
-
+
private String getInbox() throws IOException {
return ServerConfigUtils.masterInbox(conf);
}
/**
* Used for local cluster.
+ *
* @param supervisor {@link org.apache.storm.daemon.supervisor.Supervisor}
*/
public void addSupervisor(org.apache.storm.daemon.supervisor.Supervisor supervisor) {
assignmentsDistributer.addLocalSupervisor(supervisor);
}
-
+
void delayEvent(String topoId, int delaySecs, TopologyActions event, Object args) {
LOG.info("Delaying event {} for {} secs for {}", event, delaySecs, topoId);
timer.schedule(delaySecs, () -> {
@@ -1298,11 +1337,11 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
StormBase updated = new StormBase();
updated.set_topology_action_options(null);
updated.set_component_debug(Collections.emptyMap());
-
+
if (rbo.is_set_num_executors()) {
updated.set_component_executors(rbo.get_num_executors());
}
-
+
if (rbo.is_set_num_workers()) {
updated.set_num_workers(rbo.get_num_workers());
}
@@ -1310,12 +1349,12 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
updateBlobStore(topoId, rbo, ServerUtils.principalNameToSubject(rbo.get_principal()));
mkAssignments(topoId);
}
-
+
private String toTopoId(String topoName) throws NotAliveException {
return stormClusterState.getTopoId(topoName)
- .orElseThrow(() -> new NotAliveException(topoName + " is not alive"));
+ .orElseThrow(() -> new NotAliveException(topoName + " is not alive"));
}
-
+
private void transitionName(String topoName, TopologyActions event, Object eventArg, boolean errorOnNoTransition) throws Exception {
transition(toTopoId(topoName), event, eventArg, errorOnNoTransition);
}
@@ -1323,12 +1362,12 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
private void transition(String topoId, TopologyActions event, Object eventArg) throws Exception {
transition(topoId, event, eventArg, false);
}
-
+
private void transition(String topoId, TopologyActions event, Object eventArg, boolean errorOnNoTransition)
throws Exception {
LOG.info("TRANSITION: {} {} {} {}", topoId, event, eventArg, errorOnNoTransition);
assertIsLeader();
- synchronized(submitLock) {
+ synchronized (submitLock) {
IStormClusterState clusterState = stormClusterState;
StormBase base = clusterState.stormBase(topoId, null);
if (base == null || base.get_status() == null) {
@@ -1341,7 +1380,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
if (errorOnNoTransition) {
throw new RuntimeException(message);
}
-
+
if (TopologyActions.STARTUP != event) {
//STARTUP is a system event so don't log an issue
LOG.info(message);
@@ -1355,9 +1394,9 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
}
}
-
+
private void setupStormCode(Map<String, Object> conf, String topoId, String tmpJarLocation,
- Map<String, Object> topoConf, StormTopology topology) throws Exception {
+ Map<String, Object> topoConf, StormTopology topology) throws Exception {
Subject subject = getSubject();
IStormClusterState clusterState = stormClusterState;
BlobStore store = blobStore;
@@ -1398,7 +1437,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
throws AuthorizationException, IOException, KeyNotFoundException {
Map<String, Object> topoConf = new HashMap<>(topoCache.readTopoConf(topoId, subject)); //Copy the data
topoConf.putAll(configOverride);
- topoCache.updateTopoConf(topoId, subject, topoConf);
+ topoCache.updateTopoConf(topoId, subject, topoConf);
}
private void updateBlobStore(String topoId, RebalanceOptions rbo, Subject subject)
@@ -1412,7 +1451,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
updateTopologyConf(topoId, Utils.parseJson(confOverride), subject);
}
}
-
+
private Integer getBlobReplicationCount(String key) throws Exception {
BlobStore store = blobStore;
if (store != null) {
@@ -1420,7 +1459,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
return null;
}
-
+
private void waitForDesiredCodeReplication(Map<String, Object> topoConf, String topoId) throws Exception {
int minReplicationCount = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_MIN_REPLICATION_COUNT));
int maxWaitTime = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_MAX_REPLICATION_WAIT_TIME_SEC));
@@ -1434,12 +1473,12 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
//When is this ever null?
if (blobStore != null) {
while (jarCount < minReplicationCount &&
- codeCount < minReplicationCount &&
- confCount < minReplicationCount) {
+ codeCount < minReplicationCount &&
+ confCount < minReplicationCount) {
if (maxWaitTime > 0 && totalWaitTime > maxWaitTime) {
LOG.info("desired replication count of {} not achieved but we have hit the max wait time {}"
- + " so moving on with replication count for conf key = {} for code key = {} for jar key = ",
- minReplicationCount, maxWaitTime, confCount, codeCount, jarCount);
+ + " so moving on with replication count for conf key = {} for code key = {} for jar key = ",
+ minReplicationCount, maxWaitTime, confCount, codeCount, jarCount);
return;
}
LOG.debug("Checking if I am still the leader");
@@ -1456,10 +1495,10 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
}
LOG.info("desired replication count {} achieved, current-replication-count for conf key = {},"
- + " current-replication-count for code key = {}, current-replication-count for jar key = {}",
- minReplicationCount, confCount, codeCount, jarCount);
+ + " current-replication-count for code key = {}, current-replication-count for jar key = {}",
+ minReplicationCount, confCount, codeCount, jarCount);
}
-
+
private TopologyDetails readTopologyDetails(String topoId, StormBase base) throws KeyNotFoundException,
AuthorizationException, IOException, InvalidTopologyException {
assert (base != null);
@@ -1473,44 +1512,49 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
Map<List<Integer>, String> rawExecToComponent = computeExecutorToComponent(topoId, base, topoConf, topo);
Map<ExecutorDetails, String> executorsToComponent = new HashMap<>();
- for (Entry<List<Integer>, String> entry: rawExecToComponent.entrySet()) {
+ for (Entry<List<Integer>, String> entry : rawExecToComponent.entrySet()) {
List<Integer> execs = entry.getKey();
ExecutorDetails execDetails = new ExecutorDetails(execs.get(0), execs.get(1));
executorsToComponent.put(execDetails, entry.getValue());
}
-
+
return new TopologyDetails(topoId, topoConf, topo, base.get_num_workers(), executorsToComponent, base.get_launch_time_secs(),
- base.get_owner());
+ base.get_owner());
}
private void updateHeartbeatsFromZkHeartbeat(String topoId, Set<List<Integer>> allExecutors, Assignment existingAssignment) {
LOG.debug("Updating heartbeats for {} {} (from ZK heartbeat)", topoId, allExecutors);
IStormClusterState state = stormClusterState;
- Map<List<Integer>, Map<String, Object>> executorBeats = StatsUtil.convertExecutorBeats(state.executorBeats(topoId, existingAssignment.get_executor_node_port()));
+ Map<List<Integer>, Map<String, Object>> executorBeats =
+ StatsUtil.convertExecutorBeats(state.executorBeats(topoId, existingAssignment.get_executor_node_port()));
Map<List<Integer>, Map<String, Object>> cache = StatsUtil.updateHeartbeatCacheFromZkHeartbeat(heartbeatsCache.get().get(topoId),
- executorBeats, allExecutors, ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_TASK_TIMEOUT_SECS)));
+ executorBeats, allExecutors,
+ ObjectReader.getInt(conf.get(
+ DaemonConfig
+ .NIMBUS_TASK_TIMEOUT_SECS)));
heartbeatsCache.getAndUpdate(new Assoc<>(topoId, cache));
}
private void updateHeartbeats(String topoId, Set<List<Integer>> allExecutors, Assignment existingAssignment) {
LOG.debug("Updating heartbeats for {} {}", topoId, allExecutors);
Map<List<Integer>, Map<String, Object>> cache = heartbeatsCache.get().get(topoId);
- if(cache == null) {
+ if (cache == null) {
cache = new HashMap<>();
heartbeatsCache.getAndUpdate(new Assoc<>(topoId, cache));
}
StatsUtil.updateHeartbeatCache(heartbeatsCache.get().get(topoId),
- null, allExecutors, ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_TASK_TIMEOUT_SECS)));
+ null, allExecutors, ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_TASK_TIMEOUT_SECS)));
}
/**
* Update all the heartbeats for all the topologies' executors.
+ *
* @param existingAssignments current assignments (thrift)
* @param topologyToExecutors topology ID to executors.
*/
private void updateAllHeartbeats(Map<String, Assignment> existingAssignments,
- Map<String, Set<List<Integer>>> topologyToExecutors, Set<String> zkHeartbeatTopologies) {
- for (Entry<String, Assignment> entry: existingAssignments.entrySet()) {
+ Map<String, Set<List<Integer>>> topologyToExecutors, Set<String> zkHeartbeatTopologies) {
+ for (Entry<String, Assignment> entry : existingAssignments.entrySet()) {
String topoId = entry.getKey();
if (zkHeartbeatTopologies.contains(topoId)) {
updateHeartbeatsFromZkHeartbeat(topoId, topologyToExecutors.get(topoId), entry.getValue());
@@ -1524,42 +1568,43 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
Map<List<Integer>, Map<String, Object>> executorBeats = StatsUtil.convertWorkerBeats(workerHeartbeat);
String topoId = workerHeartbeat.get_storm_id();
Map<List<Integer>, Map<String, Object>> cache = heartbeatsCache.get().get(topoId);
- if(cache == null) {
+ if (cache == null) {
cache = new HashMap<>();
heartbeatsCache.getAndUpdate(new Assoc<>(topoId, cache));
}
Set<List<Integer>> executors = new HashSet<>();
- for(ExecutorInfo executorInfo : workerHeartbeat.get_executors()) {
+ for (ExecutorInfo executorInfo : workerHeartbeat.get_executors()) {
executors.add(Arrays.asList(executorInfo.get_task_start(), executorInfo.get_task_end()));
}
StatsUtil.updateHeartbeatCache(heartbeatsCache.get().get(topoId), executorBeats, executors,
- ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_TASK_TIMEOUT_SECS)));
+ ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_TASK_TIMEOUT_SECS)));
}
private void updateCachedHeartbeatsFromSupervisor(SupervisorWorkerHeartbeats workerHeartbeats) {
workerHeartbeats.get_worker_heartbeats().forEach(this::updateCachedHeartbeatsFromWorker);
- if(!heartbeatsReadyFlag.get() && !Strings.isNullOrEmpty(workerHeartbeats.get_supervisor_id())) {
+ if (!heartbeatsReadyFlag.get() && !Strings.isNullOrEmpty(workerHeartbeats.get_supervisor_id())) {
heartbeatsRecoveryStrategy.reportNodeId(workerHeartbeats.get_supervisor_id());
}
}
/**
- * Decide if the heartbeats is recovered for a master, will wait for all the assignments nodes to recovery,
- * every node will take care its node heartbeats reporting.
+ * Decide if the heartbeats is recovered for a master, will wait for all the assignments nodes to recovery, every node will take care
+ * its node heartbeats reporting.
+ *
* @return true if all nodes have reported heartbeats or exceeds max-time-out
*/
private boolean isHeartbeatsRecovered() {
- if(heartbeatsReadyFlag.get()) {
+ if (heartbeatsReadyFlag.get()) {
return true;
}
Set<String> allNodes = new HashSet<>();
- for(Map.Entry<String, Assignment> assignmentEntry: stormClusterState.assignmentsInfo().entrySet()) {
+ for (Map.Entry<String, Assignment> assignmentEntry : stormClusterState.assignmentsInfo().entrySet()) {
allNodes.addAll(assignmentEntry.getValue().get_node_host().keySet());
}
boolean isReady = heartbeatsRecoveryStrategy.isReady(allNodes);
- if(isReady) {
+ if (isReady) {
heartbeatsReadyFlag.getAndSet(true);
}
return isReady;
@@ -1567,12 +1612,13 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
/**
* Decide if the assignments is synchronized.
+ *
* @return true if assignments have been synchronized from remote state store
*/
private boolean isAssignmentsRecovered() {
return stormClusterState.isAssignmentsBackendSynchronized();
}
-
+
private Set<List<Integer>> aliveExecutors(TopologyDetails td, Set<List<Integer>> allExecutors, Assignment assignment) {
String topoId = td.getId();
Map<List<Integer>, Map<String, Object>> hbCache = heartbeatsCache.get().get(topoId);
@@ -1581,13 +1627,13 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
hbCache = new HashMap<>();
}
LOG.debug("NEW Computing alive executors for {}\nExecutors: {}\nAssignment: {}\nHeartbeat cache: {}",
- topoId, allExecutors, assignment, hbCache);
-
+ topoId, allExecutors, assignment, hbCache);
+
int taskLaunchSecs = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_TASK_LAUNCH_SECS));
Set<List<Integer>> ret = new HashSet<>();
Map<List<Long>, Long> execToStartTimes = assignment.get_executor_start_time_secs();
- for (List<Integer> exec: allExecutors) {
+ for (List<Integer> exec : allExecutors) {
List<Long> longExec = new ArrayList<Long>(exec.size());
for (Integer num : exec) {
longExec.add(num.longValue());
@@ -1596,7 +1642,7 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
Long startTime = execToStartTimes.get(longExec);
Map<String, Object> executorCache = hbCache.get(StatsUtil.convertExecutor(longExec));
//null isTimedOut means worker never reported any heartbeat
- Boolean isTimedOut = executorCache == null ? null : (Boolean)executorCache.get("is-timed-out");
+ Boolean isTimedOut = executorCache == null ? null : (Boolean) executorCache.get("is-timed-out");
Integer delta = startTime == null ? null : Time.deltaSecs(startTime.intValue());
if (startTime != null && ((delta < taskLaunchSecs) || (isTimedOut != null && !isTimedOut))) {
ret.add(exec);
@@ -1606,9 +1652,9 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
}
return ret;
}
-
+
private List<List<Integer>> computeExecutors(String topoId, StormBase base, Map<String, Object> topoConf,
- StormTopology topology)
+ StormTopology topology)
throws KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException {
assert (base != null);
@@ -1617,13 +1663,13 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon {
if (compToExecutors != null) {
Map<Integer, String> taskInfo = StormCommon.stormTaskInfo(topology, topoConf);
Map<String, List<Integer>> compToTaskList = Utils.reverseMap(taskInfo);
- for (Entry<String, List<Integer>> entry: compToTaskList.entrySet()) {
+ for (Entry<String, List<Integer>> entry : compToTaskList.entrySet()) {
List<Integer> comps = entry.getValue();
comps.sort(null);
Integer numExecutors = compToExecutors.get(entry.getKey());
if (numExecutors != null) {
List<List<Integer>> partitioned = Utils.partitionFixed(numExecutors, comps);
- for (List<I
<TRUNCATED>
[53/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-perf
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-perf
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0e409ecd
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0e409ecd
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0e409ecd
Branch: refs/heads/master
Commit: 0e409ecd8b2d6956ed38f78bb068ce8fef67e83b
Parents: f4ba7c9
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 23:14:26 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 02:32:40 2018 -0400
----------------------------------------------------------------------
examples/storm-perf/pom.xml | 2 +-
.../perf/ConstSpoutIdBoltNullBoltTopo.java | 12 ++--
.../storm/perf/ConstSpoutNullBoltTopo.java | 4 +-
.../storm/perf/FileReadWordCountTopo.java | 33 +++++------
.../storm/perf/HdfsSpoutNullBoltTopo.java | 3 +-
.../org/apache/storm/perf/JCQueuePerfTest.java | 31 +++++-----
.../org/apache/storm/perf/JCToolsPerfTest.java | 27 ++++-----
.../apache/storm/perf/KafkaClientHdfsTopo.java | 8 +--
.../perf/KafkaClientSpoutNullBoltTopo.java | 15 ++---
.../apache/storm/perf/LowThroughputTopo.java | 9 ++-
.../storm/perf/SimplifiedWordCountTopo.java | 1 -
.../storm/perf/StrGenSpoutHdfsBoltTopo.java | 3 +-
.../org/apache/storm/perf/ThroughputMeter.java | 16 ++---
.../org/apache/storm/perf/bolt/CountBolt.java | 1 -
.../org/apache/storm/perf/bolt/DevNullBolt.java | 1 -
.../java/org/apache/storm/perf/bolt/IdBolt.java | 1 -
.../storm/perf/bolt/SplitSentenceBolt.java | 1 -
.../org/apache/storm/perf/spout/ConstSpout.java | 1 -
.../apache/storm/perf/spout/FileReadSpout.java | 1 -
.../apache/storm/perf/spout/StringGenSpout.java | 27 ++++-----
.../apache/storm/perf/spout/WordGenSpout.java | 62 ++++++++++----------
.../storm/perf/utils/BasicMetricsCollector.java | 23 ++++----
.../org/apache/storm/perf/utils/Helper.java | 7 ++-
.../apache/storm/perf/utils/IdentityBolt.java | 3 +-
.../apache/storm/perf/utils/MetricsSample.java | 3 +-
25 files changed, 139 insertions(+), 156 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/pom.xml
----------------------------------------------------------------------
diff --git a/examples/storm-perf/pom.xml b/examples/storm-perf/pom.xml
index f8eb965..4bbabfa3 100644
--- a/examples/storm-perf/pom.xml
+++ b/examples/storm-perf/pom.xml
@@ -95,7 +95,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>100</maxAllowedViolations>
+ <maxAllowedViolations>65</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java
index dc3649e..a65efe6 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutIdBoltNullBoltTopo.java
@@ -19,7 +19,6 @@
package org.apache.storm.perf;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.perf.bolt.DevNullBolt;
@@ -30,11 +29,8 @@ import org.apache.storm.topology.TopologyBuilder;
import org.apache.storm.utils.Utils;
/**
- * ConstSpout -> IdBolt -> DevNullBolt
- * This topology measures speed of messaging between spouts->bolt and bolt->bolt
- * ConstSpout : Continuously emits a constant string
- * IdBolt : clones and emits input tuples
- * DevNullBolt : discards incoming tuples
+ * ConstSpout -> IdBolt -> DevNullBolt This topology measures speed of messaging between spouts->bolt and bolt->bolt ConstSpout :
+ * Continuously emits a constant string IdBolt : clones and emits input tuples DevNullBolt : discards incoming tuples
*/
public class ConstSpoutIdBoltNullBoltTopo {
@@ -66,11 +62,11 @@ public class ConstSpoutIdBoltNullBoltTopo {
int numBolt1 = Helper.getInt(conf, BOLT1_COUNT, 1);
builder.setBolt(BOLT1_ID, bolt1, numBolt1)
- .localOrShuffleGrouping(SPOUT_ID);
+ .localOrShuffleGrouping(SPOUT_ID);
int numBolt2 = Helper.getInt(conf, BOLT2_COUNT, 1);
builder.setBolt(BOLT2_ID, bolt2, numBolt2)
- .localOrShuffleGrouping(BOLT1_ID);
+ .localOrShuffleGrouping(BOLT1_ID);
System.err.printf("====> Using : numSpouts = %d , numBolt1 = %d, numBolt2=%d\n", numSpouts, numBolt1, numBolt2);
return builder.createTopology();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java
index ee778fb..9683e08 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/ConstSpoutNullBoltTopo.java
@@ -19,7 +19,6 @@
package org.apache.storm.perf;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.perf.bolt.DevNullBolt;
@@ -89,7 +88,8 @@ public class ConstSpoutNullBoltTopo {
// For reference : numbers taken on MacBook Pro mid 2015
// -- ACKer=0: ~8 mill/sec (batchSz=2k & recvQsize=50k). 6.7 mill/sec (batchSz=1 & recvQsize=1k)
// -- ACKer=1: ~1 mill/sec, lat= ~1 microsec (batchSz=1 & bolt.wait.strategy=Park bolt.wait.park.micros=0)
- // -- ACKer=1: ~1.3 mill/sec, lat= ~11 micros (batchSz=1 & receive.buffer.size=1k, bolt.wait & bp.wait = Progressive[defaults])
+ // -- ACKer=1: ~1.3 mill/sec, lat= ~11 micros (batchSz=1 & receive.buffer.size=1k, bolt.wait & bp.wait =
+ // Progressive[defaults])
// -- ACKer=1: ~1.6 mill/sec, lat= ~300 micros (batchSz=500 & bolt.wait.strategy=Park bolt.wait.park.micros=0)
topoConf.put(Config.TOPOLOGY_SPOUT_RECVQ_SKIPS, 8);
topoConf.put(Config.TOPOLOGY_PRODUCER_BATCH_SIZE, 500);
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java
index 7e9256e..827337e 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/FileReadWordCountTopo.java
@@ -1,25 +1,24 @@
/*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
package org.apache.storm.perf;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.perf.bolt.CountBolt;
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java
index e288a5c..4f97dfa 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/HdfsSpoutNullBoltTopo.java
@@ -19,7 +19,6 @@
package org.apache.storm.perf;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.hdfs.spout.HdfsSpout;
@@ -78,7 +77,7 @@ public class HdfsSpoutNullBoltTopo {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SPOUT_ID, spout, spoutNum);
builder.setBolt(BOLT_ID, bolt, boltNum)
- .localOrShuffleGrouping(SPOUT_ID);
+ .localOrShuffleGrouping(SPOUT_ID);
return builder.createTopology();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/JCQueuePerfTest.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/JCQueuePerfTest.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/JCQueuePerfTest.java
index 700ce4e..f2bfd31 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/JCQueuePerfTest.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/JCQueuePerfTest.java
@@ -19,7 +19,6 @@
package org.apache.storm.perf;
import java.util.concurrent.locks.LockSupport;
-
import org.apache.storm.policy.WaitStrategyPark;
import org.apache.storm.utils.JCQueue;
import org.apache.storm.utils.MutableLong;
@@ -28,15 +27,15 @@ public class JCQueuePerfTest {
// Usage: Let it and then explicitly terminate.
// Metrics will be printed when application is terminated.
public static void main(String[] args) throws Exception {
-// oneProducer1Consumer(1000); // -- measurement 1
-// twoProducer1Consumer(1000); // -- measurement 2
-// threeProducer1Consumer(1); // -- measurement 3
+ // oneProducer1Consumer(1000); // -- measurement 1
+ // twoProducer1Consumer(1000); // -- measurement 2
+ // threeProducer1Consumer(1); // -- measurement 3
-// oneProducer2Consumers(); // -- measurement 4
+ // oneProducer2Consumers(); // -- measurement 4
-// producerFwdConsumer(); // -- measurement 5
+ // producerFwdConsumer(); // -- measurement 5
-// ackingProducerSimulation(); // -- measurement 6
+ // ackingProducerSimulation(); // -- measurement 6
while (true) {
Thread.sleep(1000);
@@ -46,8 +45,8 @@ public class JCQueuePerfTest {
private static void ackingProducerSimulation() {
WaitStrategyPark ws = new WaitStrategyPark(100);
- JCQueue spoutQ = new JCQueue("spoutQ", 1024, 0, 100, ws, "test", "test",1000, 1000);
- JCQueue ackQ = new JCQueue("ackQ", 1024, 0, 100, ws, "test", "test",1000, 1000);
+ JCQueue spoutQ = new JCQueue("spoutQ", 1024, 0, 100, ws, "test", "test", 1000, 1000);
+ JCQueue ackQ = new JCQueue("ackQ", 1024, 0, 100, ws, "test", "test", 1000, 1000);
final AckingProducer ackingProducer = new AckingProducer(spoutQ, ackQ);
final Acker acker = new Acker(ackQ, spoutQ);
@@ -57,8 +56,8 @@ public class JCQueuePerfTest {
private static void producerFwdConsumer(int prodBatchSz) {
WaitStrategyPark ws = new WaitStrategyPark(100);
- JCQueue q1 = new JCQueue("q1", 1024, 0, prodBatchSz, ws, "test", "test",1000, 1000);
- JCQueue q2 = new JCQueue("q2", 1024, 0, prodBatchSz, ws, "test", "test",1000, 1000);
+ JCQueue q1 = new JCQueue("q1", 1024, 0, prodBatchSz, ws, "test", "test", 1000, 1000);
+ JCQueue q2 = new JCQueue("q2", 1024, 0, prodBatchSz, ws, "test", "test", 1000, 1000);
final Producer prod = new Producer(q1);
final Forwarder fwd = new Forwarder(q1, q2);
@@ -69,7 +68,7 @@ public class JCQueuePerfTest {
private static void oneProducer1Consumer(int prodBatchSz) {
- JCQueue q1 = new JCQueue("q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test",1000, 1000);
+ JCQueue q1 = new JCQueue("q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test", 1000, 1000);
final Producer prod1 = new Producer(q1);
final Consumer cons1 = new Consumer(q1);
@@ -78,7 +77,7 @@ public class JCQueuePerfTest {
}
private static void twoProducer1Consumer(int prodBatchSz) {
- JCQueue q1 = new JCQueue("q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test",1000, 1000);
+ JCQueue q1 = new JCQueue("q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test", 1000, 1000);
final Producer prod1 = new Producer(q1);
final Producer prod2 = new Producer(q1);
@@ -88,7 +87,7 @@ public class JCQueuePerfTest {
}
private static void threeProducer1Consumer(int prodBatchSz) {
- JCQueue q1 = new JCQueue("q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test",1000, 1000);
+ JCQueue q1 = new JCQueue("q1", 50_000, 0, prodBatchSz, new WaitStrategyPark(100), "test", "test", 1000, 1000);
final Producer prod1 = new Producer(q1);
final Producer prod2 = new Producer(q1);
@@ -101,8 +100,8 @@ public class JCQueuePerfTest {
private static void oneProducer2Consumers(int prodBatchSz) {
WaitStrategyPark ws = new WaitStrategyPark(100);
- JCQueue q1 = new JCQueue("q1", 1024, 0, prodBatchSz, ws, "test", "test",1000, 1000);
- JCQueue q2 = new JCQueue("q2", 1024, 0, prodBatchSz, ws, "test", "test",1000, 1000);
+ JCQueue q1 = new JCQueue("q1", 1024, 0, prodBatchSz, ws, "test", "test", 1000, 1000);
+ JCQueue q2 = new JCQueue("q2", 1024, 0, prodBatchSz, ws, "test", "test", 1000, 1000);
final Producer2 prod1 = new Producer2(q1, q2);
final Consumer cons1 = new Consumer(q1);
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/JCToolsPerfTest.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/JCToolsPerfTest.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/JCToolsPerfTest.java
index 98525ed..a439522 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/JCToolsPerfTest.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/JCToolsPerfTest.java
@@ -19,25 +19,24 @@
package org.apache.storm.perf;
import java.util.concurrent.locks.LockSupport;
-
import org.apache.storm.utils.MutableLong;
import org.jctools.queues.MpscArrayQueue;
public class JCToolsPerfTest {
public static void main(String[] args) throws Exception {
-// oneProducer1Consumer();
-// twoProducer1Consumer();
-// threeProducer1Consumer();
-// oneProducer2Consumers();
-// producerFwdConsumer();
-
-// JCQueue spoutQ = new JCQueue("spoutQ", 1024, 100, 0);
-// JCQueue ackQ = new JCQueue("ackQ", 1024, 100, 0);
-//
-// final AckingProducer ackingProducer = new AckingProducer(spoutQ, ackQ);
-// final Acker acker = new Acker(ackQ, spoutQ);
-//
-// runAllThds(ackingProducer, acker);
+ // oneProducer1Consumer();
+ // twoProducer1Consumer();
+ // threeProducer1Consumer();
+ // oneProducer2Consumers();
+ // producerFwdConsumer();
+
+ // JCQueue spoutQ = new JCQueue("spoutQ", 1024, 100, 0);
+ // JCQueue ackQ = new JCQueue("ackQ", 1024, 100, 0);
+ //
+ // final AckingProducer ackingProducer = new AckingProducer(spoutQ, ackQ);
+ // final Acker acker = new Acker(ackQ, spoutQ);
+ //
+ // runAllThds(ackingProducer, acker);
while (true) {
Thread.sleep(1000);
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java
index 0432d9e..67117e4 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientHdfsTopo.java
@@ -19,7 +19,6 @@
package org.apache.storm.perf;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.hdfs.bolt.HdfsBolt;
@@ -80,8 +79,9 @@ public class KafkaClientHdfsTopo {
String topicName = getStr(config, KAFKA_TOPIC);
KafkaSpoutConfig<String, String> spoutConfig = KafkaSpoutConfig.builder(bootstrapHosts, topicName)
- .setFirstPollOffsetStrategy(KafkaSpoutConfig.FirstPollOffsetStrategy.EARLIEST)
- .build();
+ .setFirstPollOffsetStrategy(
+ KafkaSpoutConfig.FirstPollOffsetStrategy.EARLIEST)
+ .build();
KafkaSpout<String, String> spout = new KafkaSpout<>(spoutConfig);
@@ -106,7 +106,7 @@ public class KafkaClientHdfsTopo {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SPOUT_ID, spout, spoutNum);
builder.setBolt(BOLT_ID, bolt, boltNum)
- .localOrShuffleGrouping(SPOUT_ID);
+ .localOrShuffleGrouping(SPOUT_ID);
return builder.createTopology();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java
index 4d88702..a75785d 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/KafkaClientSpoutNullBoltTopo.java
@@ -65,15 +65,16 @@ public class KafkaClientSpoutNullBoltTopo {
String kafkaTopic = Optional.ofNullable(Helper.getStr(config, KAFKA_TOPIC)).orElse("storm-perf-null-bolt-topic");
ProcessingGuarantee processingGuarantee = ProcessingGuarantee.valueOf(
Optional.ofNullable(Helper.getStr(config, PROCESSING_GUARANTEE))
- .orElse(ProcessingGuarantee.AT_LEAST_ONCE.name()));
+ .orElse(ProcessingGuarantee.AT_LEAST_ONCE.name()));
int offsetCommitPeriodMs = Helper.getInt(config, OFFSET_COMMIT_PERIOD_MS, 30_000);
KafkaSpoutConfig<String, String> kafkaSpoutConfig = KafkaSpoutConfig.builder(bootstrapServers, kafkaTopic)
- .setProcessingGuarantee(processingGuarantee)
- .setOffsetCommitPeriodMs(offsetCommitPeriodMs)
- .setFirstPollOffsetStrategy(KafkaSpoutConfig.FirstPollOffsetStrategy.EARLIEST)
- .setTupleTrackingEnforced(true)
- .build();
+ .setProcessingGuarantee(processingGuarantee)
+ .setOffsetCommitPeriodMs(offsetCommitPeriodMs)
+ .setFirstPollOffsetStrategy(
+ KafkaSpoutConfig.FirstPollOffsetStrategy.EARLIEST)
+ .setTupleTrackingEnforced(true)
+ .build();
KafkaSpout<String, String> spout = new KafkaSpout<>(kafkaSpoutConfig);
@@ -84,7 +85,7 @@ public class KafkaClientSpoutNullBoltTopo {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SPOUT_ID, spout, spoutNum);
builder.setBolt(BOLT_ID, bolt, boltNum)
- .localOrShuffleGrouping(SPOUT_ID);
+ .localOrShuffleGrouping(SPOUT_ID);
return builder.createTopology();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java
index 5560780..e4972a4 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/LowThroughputTopo.java
@@ -22,7 +22,6 @@ package org.apache.storm.perf;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.perf.utils.Helper;
@@ -64,7 +63,7 @@ public class LowThroughputTopo {
BoltDeclarer bd = builder.setBolt(BOLT_ID, bolt, Helper.getInt(conf, BOLT_COUNT, 1));
bd.localOrShuffleGrouping(SPOUT_ID);
-// bd.shuffleGrouping(SPOUT_ID);
+ // bd.shuffleGrouping(SPOUT_ID);
return builder.createTopology();
}
@@ -111,7 +110,7 @@ public class LowThroughputTopo {
@Override
public void open(Map<String, Object> conf, TopologyContext context,
- SpoutOutputCollector collector) {
+ SpoutOutputCollector collector) {
this.collector = collector;
}
@@ -135,8 +134,8 @@ public class LowThroughputTopo {
@Override
public void prepare(Map<String, Object> topoConf,
- TopologyContext context,
- OutputCollector collector) {
+ TopologyContext context,
+ OutputCollector collector) {
this.collector = collector;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java
index ea5be88..0a9148c 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/SimplifiedWordCountTopo.java
@@ -19,7 +19,6 @@
package org.apache.storm.perf;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.perf.bolt.CountBolt;
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java
index a3650c6..110bf79 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/StrGenSpoutHdfsBoltTopo.java
@@ -20,7 +20,6 @@
package org.apache.storm.perf;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.hdfs.bolt.HdfsBolt;
@@ -96,7 +95,7 @@ public class StrGenSpoutHdfsBoltTopo {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout(SPOUT_ID, spout, spoutNum);
builder.setBolt(BOLT_ID, bolt, boltNum)
- .localOrShuffleGrouping(SPOUT_ID);
+ .localOrShuffleGrouping(SPOUT_ID);
return builder.createTopology();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java
index 09aec7d..bd5d1e1 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/ThroughputMeter.java
@@ -30,6 +30,14 @@ public class ThroughputMeter {
this.startTime = System.currentTimeMillis();
}
+ /**
+ * @return events/sec
+ */
+ private static double calcThroughput(long count, long startTime, long endTime) {
+ long gap = (endTime - startTime);
+ return (count / gap) * 1000;
+ }
+
public String getName() {
return name;
}
@@ -61,12 +69,4 @@ public class ThroughputMeter {
count = 0;
return result;
}
-
- /**
- * @return events/sec
- */
- private static double calcThroughput(long count, long startTime, long endTime) {
- long gap = (endTime - startTime);
- return (count / gap) * 1000;
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java
index 368699b..ee1bf7e 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/CountBolt.java
@@ -20,7 +20,6 @@ package org.apache.storm.perf.bolt;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java
index 5f9c710..abb397d 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/DevNullBolt.java
@@ -20,7 +20,6 @@ package org.apache.storm.perf.bolt;
import java.util.Map;
import java.util.concurrent.locks.LockSupport;
-
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java
index 0644e31..5387499 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/IdBolt.java
@@ -19,7 +19,6 @@
package org.apache.storm.perf.bolt;
import java.util.Map;
-
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java
index abb5af8..85a3f5a 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/bolt/SplitSentenceBolt.java
@@ -19,7 +19,6 @@
package org.apache.storm.perf.bolt;
import java.util.Map;
-
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java
index 46f12ab..656f992 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/ConstSpout.java
@@ -20,7 +20,6 @@ package org.apache.storm.perf.spout;
import java.util.ArrayList;
import java.util.Map;
-
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java
index 4f27d3b..a819fa2 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/FileReadSpout.java
@@ -27,7 +27,6 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java
index 530f7b6..01964ea 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/StringGenSpout.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-
import org.apache.commons.lang.RandomStringUtils;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -30,18 +29,18 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
-/** Spout pre-computes a list with 30k fixed length random strings.
- * Emits sequentially from this list, over and over again.
+/**
+ * Spout pre-computes a list with 30k fixed length random strings. Emits sequentially from this list, over and over again.
*/
public class StringGenSpout extends BaseRichSpout {
private static final String DEFAULT_FIELD_NAME = "str";
- private int strLen;
private final int strCount = 30_000;
+ ArrayList<String> records;
+ private int strLen;
private String fieldName = DEFAULT_FIELD_NAME;
private SpoutOutputCollector collector = null;
- ArrayList<String> records;
private int curr = 0;
private int count = 0;
@@ -49,6 +48,14 @@ public class StringGenSpout extends BaseRichSpout {
this.strLen = strLen;
}
+ private static ArrayList<String> genStringList(int strLen, int count) {
+ ArrayList<String> result = new ArrayList<String>(count);
+ for (int i = 0; i < count; i++) {
+ result.add(RandomStringUtils.random(strLen));
+ }
+ return result;
+ }
+
public StringGenSpout withFieldName(String fieldName) {
this.fieldName = fieldName;
return this;
@@ -66,18 +73,10 @@ public class StringGenSpout extends BaseRichSpout {
this.collector = collector;
}
- private static ArrayList<String> genStringList(int strLen, int count) {
- ArrayList<String> result = new ArrayList<String>(count);
- for (int i = 0; i < count; i++) {
- result.add( RandomStringUtils.random(strLen) );
- }
- return result;
- }
-
@Override
public void nextTuple() {
List<Object> tuple;
- if(curr < strCount) {
+ if (curr < strCount) {
tuple = Collections.singletonList((Object) records.get(curr));
++curr;
collector.emit(tuple, ++count);
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java
index b3195c1..f46d6f3 100644
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/spout/WordGenSpout.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Map;
-
+import org.apache.storm.perf.ThroughputMeter;
import org.apache.storm.perf.utils.Helper;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -32,7 +32,6 @@ import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
-import org.apache.storm.perf.ThroughputMeter;
public class WordGenSpout extends BaseRichSpout {
public static final String FIELDS = "word";
@@ -51,10 +50,37 @@ public class WordGenSpout extends BaseRichSpout {
this.file = file;
}
+ /**
+ * Reads text file and extracts words from each line.
+ *
+ * @return a list of all (non-unique) words
+ */
+ public static ArrayList<String> readWords(String file) {
+ ArrayList<String> lines = new ArrayList<>();
+ try {
+ FileInputStream input = new FileInputStream(file);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(input));
+ try {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ for (String word : line.split("\\s+"))
+ lines.add(word);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Reading file failed", e);
+ } finally {
+ reader.close();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Error closing reader", e);
+ }
+ return lines;
+ }
+
@Override
public void open(Map<String, Object> conf,
- TopologyContext context,
- SpoutOutputCollector collector) {
+ TopologyContext context,
+ SpoutOutputCollector collector) {
this.collector = collector;
Integer ackers = Helper.getInt(conf, "topology.acker.executors", 0);
if (ackers.equals(0)) {
@@ -67,7 +93,7 @@ public class WordGenSpout extends BaseRichSpout {
@Override
public void nextTuple() {
- index = (index < words.size()-1) ? index+1 : 0;
+ index = (index < words.size() - 1) ? index + 1 : 0;
String word = words.get(index);
if (ackEnabled) {
collector.emit(new Values(word), count);
@@ -84,30 +110,4 @@ public class WordGenSpout extends BaseRichSpout {
declarer.declare(new Fields(FIELDS));
}
- /**
- * Reads text file and extracts words from each line.
- * @return a list of all (non-unique) words
- */
- public static ArrayList<String> readWords(String file) {
- ArrayList<String> lines = new ArrayList<>();
- try {
- FileInputStream input = new FileInputStream(file);
- BufferedReader reader = new BufferedReader(new InputStreamReader(input));
- try {
- String line;
- while ((line = reader.readLine()) != null) {
- for (String word : line.split("\\s+"))
- lines.add(word);
- }
- } catch (IOException e) {
- throw new RuntimeException("Reading file failed", e);
- } finally {
- reader.close();
- }
- } catch (IOException e) {
- throw new RuntimeException("Error closing reader", e);
- }
- return lines;
- }
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java
index 7cfd354..735abe0 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/BasicMetricsCollector.java
@@ -26,7 +26,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
import org.apache.log4j.Logger;
import org.apache.storm.generated.Nimbus;
import org.apache.storm.utils.Utils;
@@ -160,9 +159,9 @@ public class BasicMetricsCollector implements AutoCloseable {
this.maxLatency = latency;
}
metrics.put(SPOUT_AVG_COMPLETE_LATENCY,
- String.format(SPOUT_AVG_LATENCY_FORMAT, latency));
+ String.format(SPOUT_AVG_LATENCY_FORMAT, latency));
metrics.put(SPOUT_MAX_COMPLETE_LATENCY,
- String.format(SPOUT_MAX_LATENCY_FORMAT, this.maxLatency));
+ String.format(SPOUT_MAX_LATENCY_FORMAT, this.maxLatency));
}
}
@@ -192,10 +191,12 @@ public class BasicMetricsCollector implements AutoCloseable {
header.add(SPOUT_MAX_COMPLETE_LATENCY);
}
- writer.println("\n------------------------------------------------------------------------------------------------------------------");
+ writer.println(
+ "\n------------------------------------------------------------------------------------------------------------------");
String str = Utils.join(header, ",");
writer.println(str);
- writer.println("------------------------------------------------------------------------------------------------------------------");
+ writer
+ .println("------------------------------------------------------------------------------------------------------------------");
writer.flush();
}
@@ -211,28 +212,28 @@ public class BasicMetricsCollector implements AutoCloseable {
boolean collectTopologyStats(Set<MetricsItem> items) {
return items.contains(MetricsItem.ALL)
- || items.contains(MetricsItem.TOPOLOGY_STATS);
+ || items.contains(MetricsItem.TOPOLOGY_STATS);
}
boolean collectExecutorStats(Set<MetricsItem> items) {
return items.contains(MetricsItem.ALL)
- || items.contains(MetricsItem.XSFER_RATE)
- || items.contains(MetricsItem.SPOUT_LATENCY);
+ || items.contains(MetricsItem.XSFER_RATE)
+ || items.contains(MetricsItem.SPOUT_LATENCY);
}
boolean collectThroughput(Set<MetricsItem> items) {
return items.contains(MetricsItem.ALL)
- || items.contains(MetricsItem.XSFER_RATE);
+ || items.contains(MetricsItem.XSFER_RATE);
}
boolean collectSpoutThroughput(Set<MetricsItem> items) {
return items.contains(MetricsItem.ALL)
- || items.contains(MetricsItem.SPOUT_THROUGHPUT);
+ || items.contains(MetricsItem.SPOUT_THROUGHPUT);
}
boolean collectSpoutLatency(Set<MetricsItem> items) {
return items.contains(MetricsItem.ALL)
- || items.contains(MetricsItem.SPOUT_LATENCY);
+ || items.contains(MetricsItem.SPOUT_LATENCY);
}
public enum MetricsItem {
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java
index 4429f4c..a0f838e 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/Helper.java
@@ -19,7 +19,6 @@
package org.apache.storm.perf.utils;
import java.util.Map;
-
import org.apache.storm.StormSubmitter;
import org.apache.storm.generated.KillOptions;
import org.apache.storm.generated.Nimbus;
@@ -68,7 +67,9 @@ public class Helper {
}
}
- /** Kill topo on Ctrl-C */
+ /**
+ * Kill topo on Ctrl-C
+ */
public static void setupShutdownHook(final String topoName) {
Map<String, Object> clusterConf = Utils.readStormConfig();
final Nimbus.Iface client = NimbusClient.getConfiguredClient(clusterConf).getClient();
@@ -86,7 +87,7 @@ public class Helper {
}
public static void runOnClusterAndPrintMetrics(int durationSec, String topoName, Map<String, Object> topoConf, StormTopology topology)
- throws Exception {
+ throws Exception {
// submit topology
StormSubmitter.submitTopologyWithProgressBar(topoName, topoConf, topology);
setupShutdownHook(topoName); // handle Ctrl-C
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java
index f950203..df27263 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/IdentityBolt.java
@@ -19,7 +19,6 @@
package org.apache.storm.perf.utils;
import java.util.Map;
-
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -36,7 +35,7 @@ public class IdentityBolt extends BaseRichBolt {
@Override
public void execute(Tuple tuple) {
- collector.emit(tuple, tuple.getValues() );
+ collector.emit(tuple, tuple.getValues());
collector.ack(tuple);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/0e409ecd/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java
----------------------------------------------------------------------
diff --git a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java
index f462bc7..b70d06a 100755
--- a/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java
+++ b/examples/storm-perf/src/main/java/org/apache/storm/perf/utils/MetricsSample.java
@@ -20,7 +20,6 @@ package org.apache.storm.perf.utils;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.generated.ClusterSummary;
import org.apache.storm.generated.ExecutorSpecificStats;
import org.apache.storm.generated.ExecutorStats;
@@ -160,7 +159,7 @@ public class MetricsSample {
ret.spoutEmitted = spoutEmitted;
ret.spoutTransferred = spoutTransferred;
ret.sampleTime = System.currentTimeMillis();
-// ret.numSupervisors = clusterSummary.get_supervisors_size();
+ // ret.numSupervisors = clusterSummary.get_supervisors_size();
ret.numWorkers = 0;
ret.numExecutors = 0;
ret.numTasks = 0;
[58/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-jdbc
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-jdbc
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6d20c4af
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6d20c4af
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6d20c4af
Branch: refs/heads/master
Commit: 6d20c4af585611c6d317ad817b0a0b4b172a40ce
Parents: 53adebc
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 23:26:15 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 02:32:42 2018 -0400
----------------------------------------------------------------------
external/storm-jdbc/pom.xml | 2 +-
.../storm/jdbc/bolt/AbstractJdbcBolt.java | 55 ++++----
.../apache/storm/jdbc/bolt/JdbcInsertBolt.java | 42 +++---
.../apache/storm/jdbc/bolt/JdbcLookupBolt.java | 29 ++--
.../org/apache/storm/jdbc/common/Column.java | 27 ++--
.../storm/jdbc/common/ConnectionProvider.java | 19 +--
.../jdbc/common/HikariCPConnectionProvider.java | 30 ++--
.../apache/storm/jdbc/common/JdbcClient.java | 39 ++---
.../java/org/apache/storm/jdbc/common/Util.java | 19 +--
.../storm/jdbc/mapper/JdbcLookupMapper.java | 24 ++--
.../apache/storm/jdbc/mapper/JdbcMapper.java | 24 ++--
.../jdbc/mapper/SimpleJdbcLookupMapper.java | 36 ++---
.../storm/jdbc/mapper/SimpleJdbcMapper.java | 57 ++++----
.../storm/jdbc/trident/state/JdbcQuery.java | 24 ++--
.../storm/jdbc/trident/state/JdbcState.java | 141 +++++++++----------
.../jdbc/trident/state/JdbcStateFactory.java | 24 ++--
.../storm/jdbc/trident/state/JdbcUpdater.java | 24 ++--
.../storm/jdbc/bolt/JdbcInsertBoltTest.java | 28 ++--
.../storm/jdbc/bolt/JdbcLookupBoltTest.java | 29 ++--
.../storm/jdbc/common/JdbcClientTest.java | 63 ++++-----
.../org/apache/storm/jdbc/common/UtilTest.java | 22 ++-
21 files changed, 323 insertions(+), 435 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/pom.xml b/external/storm-jdbc/pom.xml
index 759ddae..3a09b1b 100644
--- a/external/storm-jdbc/pom.xml
+++ b/external/storm-jdbc/pom.xml
@@ -92,7 +92,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>149</maxAllowedViolations>
+ <maxAllowedViolations>36</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/AbstractJdbcBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/AbstractJdbcBolt.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/AbstractJdbcBolt.java
index 9ae455c..8aa262c 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/AbstractJdbcBolt.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/AbstractJdbcBolt.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.jdbc.bolt;
@@ -32,14 +26,7 @@ import org.slf4j.LoggerFactory;
public abstract class AbstractJdbcBolt extends BaseTickTupleAwareRichBolt {
private static final Logger LOG = LoggerFactory.getLogger(
- AbstractJdbcBolt.class);
-
- protected OutputCollector collector;
-
- protected transient JdbcClient jdbcClient;
- protected String configKey;
- protected Integer queryTimeoutSecs;
- protected ConnectionProvider connectionProvider;
+ AbstractJdbcBolt.class);
static {
/*
@@ -58,6 +45,22 @@ public abstract class AbstractJdbcBolt extends BaseTickTupleAwareRichBolt {
DriverManager.getDrivers();
}
+ protected OutputCollector collector;
+ protected transient JdbcClient jdbcClient;
+ protected String configKey;
+ protected Integer queryTimeoutSecs;
+ protected ConnectionProvider connectionProvider;
+
+ /**
+ * Constructor.
+ * <p/>
+ * @param connectionProviderParam database connection provider
+ */
+ public AbstractJdbcBolt(final ConnectionProvider connectionProviderParam) {
+ Validate.notNull(connectionProviderParam);
+ this.connectionProvider = connectionProviderParam;
+ }
+
/**
* Subclasses should call this to ensure output collector and connection
* provider are set up, and finally jdbcClient is initialized properly.
@@ -73,7 +76,7 @@ public abstract class AbstractJdbcBolt extends BaseTickTupleAwareRichBolt {
if (queryTimeoutSecs == null) {
String msgTimeout = map.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)
- .toString();
+ .toString();
queryTimeoutSecs = Integer.parseInt(msgTimeout);
}
@@ -81,16 +84,6 @@ public abstract class AbstractJdbcBolt extends BaseTickTupleAwareRichBolt {
}
/**
- * Constructor.
- * <p/>
- * @param connectionProviderParam database connection provider
- */
- public AbstractJdbcBolt(final ConnectionProvider connectionProviderParam) {
- Validate.notNull(connectionProviderParam);
- this.connectionProvider = connectionProviderParam;
- }
-
- /**
* Cleanup.
* <p/>
* Subclasses should call this to ensure connection provider can be
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcInsertBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcInsertBolt.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcInsertBolt.java
index 897ece0..77f3c6b 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcInsertBolt.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcInsertBolt.java
@@ -1,38 +1,32 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.bolt;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.tuple.Tuple;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import org.apache.commons.lang.Validate;
import org.apache.commons.lang3.StringUtils;
import org.apache.storm.jdbc.common.Column;
import org.apache.storm.jdbc.common.ConnectionProvider;
import org.apache.storm.jdbc.mapper.JdbcMapper;
-import org.apache.storm.utils.TupleUtils;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.tuple.Tuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
/**
* Basic bolt for writing to any Database table.
* <p/>
@@ -45,7 +39,7 @@ public class JdbcInsertBolt extends AbstractJdbcBolt {
private String insertQuery;
private JdbcMapper jdbcMapper;
- public JdbcInsertBolt(ConnectionProvider connectionProvider, JdbcMapper jdbcMapper) {
+ public JdbcInsertBolt(ConnectionProvider connectionProvider, JdbcMapper jdbcMapper) {
super(connectionProvider);
Validate.notNull(jdbcMapper);
@@ -76,7 +70,7 @@ public class JdbcInsertBolt extends AbstractJdbcBolt {
@Override
public void prepare(Map<String, Object> map, TopologyContext topologyContext, OutputCollector collector) {
super.prepare(map, topologyContext, collector);
- if(StringUtils.isBlank(tableName) && StringUtils.isBlank(insertQuery)) {
+ if (StringUtils.isBlank(tableName) && StringUtils.isBlank(insertQuery)) {
throw new IllegalArgumentException("You must supply either a tableName or an insert Query.");
}
}
@@ -87,7 +81,7 @@ public class JdbcInsertBolt extends AbstractJdbcBolt {
List<Column> columns = jdbcMapper.getColumns(tuple);
List<List<Column>> columnLists = new ArrayList<List<Column>>();
columnLists.add(columns);
- if(!StringUtils.isBlank(tableName)) {
+ if (!StringUtils.isBlank(tableName)) {
this.jdbcClient.insert(this.tableName, columnLists);
} else {
this.jdbcClient.executeInsertQuery(this.insertQuery, columnLists);
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcLookupBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcLookupBolt.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcLookupBolt.java
index 9589edd..ace4089 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcLookupBolt.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/bolt/JdbcLookupBolt.java
@@ -1,35 +1,28 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.bolt;
-import org.apache.storm.topology.OutputFieldsDeclarer;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.tuple.Values;
+import java.util.List;
import org.apache.commons.lang.Validate;
import org.apache.storm.jdbc.common.Column;
import org.apache.storm.jdbc.common.ConnectionProvider;
import org.apache.storm.jdbc.mapper.JdbcLookupMapper;
-import org.apache.storm.utils.TupleUtils;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.List;
-
/**
* Basic bolt for querying from any database.
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Column.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Column.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Column.java
index 73bc0fd..c796d3b 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Column.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Column.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.common;
@@ -103,9 +98,9 @@ public class Column<T> implements Serializable {
@Override
public String toString() {
return "Column{" +
- "columnName='" + columnName + '\'' +
- ", val=" + val +
- ", sqlType=" + sqlType +
- '}';
+ "columnName='" + columnName + '\'' +
+ ", val=" + val +
+ ", sqlType=" + sqlType +
+ '}';
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/ConnectionProvider.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/ConnectionProvider.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/ConnectionProvider.java
index 0232ba2..6804b5d 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/ConnectionProvider.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/ConnectionProvider.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.common;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/HikariCPConnectionProvider.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/HikariCPConnectionProvider.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/HikariCPConnectionProvider.java
index 3f3151d..dbb0009 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/HikariCPConnectionProvider.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/HikariCPConnectionProvider.java
@@ -1,30 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.common;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
-
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Map;
import java.util.Properties;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,14 +33,13 @@ public class HikariCPConnectionProvider implements ConnectionProvider {
@Override
public synchronized void prepare() {
- if(dataSource == null) {
+ if (dataSource == null) {
Properties properties = new Properties();
properties.putAll(configMap);
HikariConfig config = new HikariConfig(properties);
- if(properties.containsKey("dataSource.url")) {
+ if (properties.containsKey("dataSource.url")) {
LOG.info("DataSource Url: " + properties.getProperty("dataSource.url"));
- }
- else if (config.getJdbcUrl() != null) {
+ } else if (config.getJdbcUrl() != null) {
LOG.info("JDBC Url: " + config.getJdbcUrl());
}
this.dataSource = new HikariDataSource(config);
@@ -66,7 +58,7 @@ public class HikariCPConnectionProvider implements ConnectionProvider {
@Override
public void cleanup() {
- if(dataSource != null) {
+ if (dataSource != null) {
dataSource.close();
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/JdbcClient.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/JdbcClient.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/JdbcClient.java
index f591236..76233cc 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/JdbcClient.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/JdbcClient.java
@@ -1,34 +1,39 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.common;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.sql.*;
-import java.sql.Date;
-import java.util.*;
-
public class JdbcClient {
private static final Logger LOG = LoggerFactory.getLogger(JdbcClient.class);
@@ -51,7 +56,7 @@ public class JdbcClient {
try {
connection = connectionProvider.getConnection();
boolean autoCommit = connection.getAutoCommit();
- if(autoCommit) {
+ if (autoCommit) {
connection.setAutoCommit(false);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Util.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Util.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Util.java
index 7bac4c8..5b83ef6 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Util.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/common/Util.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.common;
import java.lang.reflect.Field;
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcLookupMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcLookupMapper.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcLookupMapper.java
index 462bee9..341dedf 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcLookupMapper.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcLookupMapper.java
@@ -1,28 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.mapper;
+import java.util.List;
+import org.apache.storm.jdbc.common.Column;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.ITuple;
import org.apache.storm.tuple.Values;
-import org.apache.storm.jdbc.common.Column;
-
-import java.util.List;
public interface JdbcLookupMapper extends JdbcMapper {
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcMapper.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcMapper.java
index b18af32..8cfa2a4 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcMapper.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/JdbcMapper.java
@@ -1,27 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.jdbc.mapper;
-import org.apache.storm.tuple.ITuple;
-import org.apache.storm.jdbc.common.Column;
+package org.apache.storm.jdbc.mapper;
import java.io.Serializable;
import java.util.List;
+import org.apache.storm.jdbc.common.Column;
+import org.apache.storm.tuple.ITuple;
public interface JdbcMapper extends Serializable {
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcLookupMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcLookupMapper.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcLookupMapper.java
index 7fadbe9..ef141e7 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcLookupMapper.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcLookupMapper.java
@@ -1,32 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.mapper;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.lang.Validate;
+import org.apache.storm.jdbc.common.Column;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.ITuple;
import org.apache.storm.tuple.Values;
-import org.apache.commons.lang.Validate;
-import org.apache.storm.jdbc.common.Column;
-
-import java.util.ArrayList;
-import java.util.List;
public class SimpleJdbcLookupMapper extends SimpleJdbcMapper implements JdbcLookupMapper {
@@ -43,12 +37,12 @@ public class SimpleJdbcLookupMapper extends SimpleJdbcMapper implements JdbcLook
public List<Values> toTuple(ITuple input, List<Column> columns) {
Values values = new Values();
- for(String field : outputFields) {
- if(input.contains(field)) {
+ for (String field : outputFields) {
+ if (input.contains(field)) {
values.add(input.getValueByField(field));
} else {
- for(Column column : columns) {
- if(column.getColumnName().equalsIgnoreCase(field)) {
+ for (Column column : columns) {
+ if (column.getColumnName().equalsIgnoreCase(field)) {
values.add(column.getVal());
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcMapper.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcMapper.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcMapper.java
index d93c29f..e4b659c 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcMapper.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/mapper/SimpleJdbcMapper.java
@@ -1,35 +1,28 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.jdbc.mapper;
-import org.apache.storm.tuple.ITuple;
-import org.apache.commons.lang.Validate;
-import org.apache.storm.jdbc.common.Column;
-import org.apache.storm.jdbc.common.ConnectionProvider;
-import org.apache.storm.jdbc.common.JdbcClient;
-import org.apache.storm.jdbc.common.Util;
+package org.apache.storm.jdbc.mapper;
import java.sql.Date;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
+import org.apache.commons.lang.Validate;
+import org.apache.storm.jdbc.common.Column;
+import org.apache.storm.jdbc.common.ConnectionProvider;
+import org.apache.storm.jdbc.common.JdbcClient;
+import org.apache.storm.jdbc.common.Util;
+import org.apache.storm.tuple.ITuple;
public class SimpleJdbcMapper implements JdbcMapper {
@@ -53,41 +46,41 @@ public class SimpleJdbcMapper implements JdbcMapper {
@Override
public List<Column> getColumns(ITuple tuple) {
List<Column> columns = new ArrayList<Column>();
- for(Column column : schemaColumns) {
+ for (Column column : schemaColumns) {
String columnName = column.getColumnName();
Integer columnSqlType = column.getSqlType();
- if(Util.getJavaType(columnSqlType).equals(String.class)) {
+ if (Util.getJavaType(columnSqlType).equals(String.class)) {
String value = tuple.getStringByField(columnName);
columns.add(new Column(columnName, value, columnSqlType));
- } else if(Util.getJavaType(columnSqlType).equals(Short.class)) {
+ } else if (Util.getJavaType(columnSqlType).equals(Short.class)) {
Short value = tuple.getShortByField(columnName);
columns.add(new Column(columnName, value, columnSqlType));
- } else if(Util.getJavaType(columnSqlType).equals(Integer.class)) {
+ } else if (Util.getJavaType(columnSqlType).equals(Integer.class)) {
Integer value = tuple.getIntegerByField(columnName);
columns.add(new Column(columnName, value, columnSqlType));
- } else if(Util.getJavaType(columnSqlType).equals(Long.class)) {
+ } else if (Util.getJavaType(columnSqlType).equals(Long.class)) {
Long value = tuple.getLongByField(columnName);
columns.add(new Column(columnName, value, columnSqlType));
- } else if(Util.getJavaType(columnSqlType).equals(Double.class)) {
+ } else if (Util.getJavaType(columnSqlType).equals(Double.class)) {
Double value = tuple.getDoubleByField(columnName);
columns.add(new Column(columnName, value, columnSqlType));
- } else if(Util.getJavaType(columnSqlType).equals(Float.class)) {
+ } else if (Util.getJavaType(columnSqlType).equals(Float.class)) {
Float value = tuple.getFloatByField(columnName);
columns.add(new Column(columnName, value, columnSqlType));
- } else if(Util.getJavaType(columnSqlType).equals(Boolean.class)) {
+ } else if (Util.getJavaType(columnSqlType).equals(Boolean.class)) {
Boolean value = tuple.getBooleanByField(columnName);
columns.add(new Column(columnName, value, columnSqlType));
- } else if(Util.getJavaType(columnSqlType).equals(byte[].class)) {
+ } else if (Util.getJavaType(columnSqlType).equals(byte[].class)) {
byte[] value = tuple.getBinaryByField(columnName);
columns.add(new Column(columnName, value, columnSqlType));
- } else if(Util.getJavaType(columnSqlType).equals(Date.class)) {
+ } else if (Util.getJavaType(columnSqlType).equals(Date.class)) {
Long value = tuple.getLongByField(columnName);
columns.add(new Column(columnName, new Date(value), columnSqlType));
- } else if(Util.getJavaType(columnSqlType).equals(Time.class)) {
+ } else if (Util.getJavaType(columnSqlType).equals(Time.class)) {
Long value = tuple.getLongByField(columnName);
columns.add(new Column(columnName, new Time(value), columnSqlType));
- } else if(Util.getJavaType(columnSqlType).equals(Timestamp.class)) {
+ } else if (Util.getJavaType(columnSqlType).equals(Timestamp.class)) {
Long value = tuple.getLongByField(columnName);
columns.add(new Column(columnName, new Timestamp(value), columnSqlType));
} else {
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcQuery.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcQuery.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcQuery.java
index 8d014cd..f6623a0 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcQuery.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcQuery.java
@@ -1,28 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.trident.state;
-import org.apache.storm.tuple.Values;
+import java.util.List;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseQueryFunction;
import org.apache.storm.trident.tuple.TridentTuple;
-
-import java.util.List;
+import org.apache.storm.tuple.Values;
public class JdbcQuery extends BaseQueryFunction<JdbcState, List<Values>> {
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcState.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcState.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcState.java
index eabf011..1954ef3 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcState.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcState.java
@@ -1,42 +1,36 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.trident.state;
-import org.apache.storm.Config;
-import org.apache.storm.topology.FailedException;
-import org.apache.storm.tuple.Values;
import com.google.common.collect.Lists;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import org.apache.commons.lang3.StringUtils;
+import org.apache.storm.Config;
import org.apache.storm.jdbc.common.Column;
import org.apache.storm.jdbc.common.ConnectionProvider;
import org.apache.storm.jdbc.common.JdbcClient;
-import org.apache.storm.jdbc.mapper.JdbcMapper;
import org.apache.storm.jdbc.mapper.JdbcLookupMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.storm.jdbc.mapper.JdbcMapper;
+import org.apache.storm.topology.FailedException;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.tuple.TridentTuple;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import org.apache.storm.tuple.Values;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class JdbcState implements State {
@@ -51,60 +45,16 @@ public class JdbcState implements State {
this.map = map;
}
- public static class Options implements Serializable {
- private JdbcMapper mapper;
- private JdbcLookupMapper jdbcLookupMapper;
- private ConnectionProvider connectionProvider;
- private String tableName;
- private String insertQuery;
- private String selectQuery;
- private Integer queryTimeoutSecs;
-
- public Options withConnectionProvider(ConnectionProvider connectionProvider) {
- this.connectionProvider = connectionProvider;
- return this;
- }
-
- public Options withTableName(String tableName) {
- this.tableName = tableName;
- return this;
- }
-
- public Options withInsertQuery(String insertQuery) {
- this.insertQuery = insertQuery;
- return this;
- }
-
- public Options withMapper(JdbcMapper mapper) {
- this.mapper = mapper;
- return this;
- }
-
- public Options withJdbcLookupMapper(JdbcLookupMapper jdbcLookupMapper) {
- this.jdbcLookupMapper = jdbcLookupMapper;
- return this;
- }
-
- public Options withSelectQuery(String selectQuery) {
- this.selectQuery = selectQuery;
- return this;
- }
-
- public Options withQueryTimeoutSecs(int queryTimeoutSecs) {
- this.queryTimeoutSecs = queryTimeoutSecs;
- return this;
- }
- }
-
protected void prepare() {
options.connectionProvider.prepare();
- if(StringUtils.isBlank(options.insertQuery) && StringUtils.isBlank(options.tableName) && StringUtils.isBlank(options.selectQuery)) {
+ if (StringUtils.isBlank(options.insertQuery) && StringUtils.isBlank(options.tableName) &&
+ StringUtils.isBlank(options.selectQuery)) {
throw new IllegalArgumentException("If you are trying to insert into DB you must supply either insertQuery or tableName." +
- "If you are attempting to user a query state you must supply a select query.");
+ "If you are attempting to user a query state you must supply a select query.");
}
- if(options.queryTimeoutSecs == null) {
+ if (options.queryTimeoutSecs == null) {
options.queryTimeoutSecs = Integer.parseInt(map.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS).toString());
}
@@ -129,7 +79,7 @@ public class JdbcState implements State {
}
try {
- if(!StringUtils.isBlank(options.tableName)) {
+ if (!StringUtils.isBlank(options.tableName)) {
jdbcClient.insert(options.tableName, columnsLists);
} else {
jdbcClient.executeInsertQuery(options.insertQuery, columnsLists);
@@ -146,7 +96,7 @@ public class JdbcState implements State {
for (TridentTuple tuple : tridentTuples) {
List<Column> columns = options.jdbcLookupMapper.getColumns(tuple);
List<List<Column>> rows = jdbcClient.select(options.selectQuery, columns);
- for(List<Column> row : rows) {
+ for (List<Column> row : rows) {
List<Values> values = options.jdbcLookupMapper.toTuple(tuple, row);
batchRetrieveResult.add(values);
}
@@ -157,4 +107,49 @@ public class JdbcState implements State {
}
return batchRetrieveResult;
}
+
+ public static class Options implements Serializable {
+ private JdbcMapper mapper;
+ private JdbcLookupMapper jdbcLookupMapper;
+ private ConnectionProvider connectionProvider;
+ private String tableName;
+ private String insertQuery;
+ private String selectQuery;
+ private Integer queryTimeoutSecs;
+
+ public Options withConnectionProvider(ConnectionProvider connectionProvider) {
+ this.connectionProvider = connectionProvider;
+ return this;
+ }
+
+ public Options withTableName(String tableName) {
+ this.tableName = tableName;
+ return this;
+ }
+
+ public Options withInsertQuery(String insertQuery) {
+ this.insertQuery = insertQuery;
+ return this;
+ }
+
+ public Options withMapper(JdbcMapper mapper) {
+ this.mapper = mapper;
+ return this;
+ }
+
+ public Options withJdbcLookupMapper(JdbcLookupMapper jdbcLookupMapper) {
+ this.jdbcLookupMapper = jdbcLookupMapper;
+ return this;
+ }
+
+ public Options withSelectQuery(String selectQuery) {
+ this.selectQuery = selectQuery;
+ return this;
+ }
+
+ public Options withQueryTimeoutSecs(int queryTimeoutSecs) {
+ this.queryTimeoutSecs = queryTimeoutSecs;
+ return this;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcStateFactory.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcStateFactory.java
index abc169f..6e15786 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcStateFactory.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcStateFactory.java
@@ -1,28 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.trident.state;
+import java.util.Map;
import org.apache.storm.task.IMetricsContext;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.state.StateFactory;
-import java.util.Map;
-
public class JdbcStateFactory implements StateFactory {
private JdbcState.Options options;
@@ -33,7 +27,7 @@ public class JdbcStateFactory implements StateFactory {
@Override
public State makeState(Map<String, Object> map, IMetricsContext iMetricsContext, int partitionIndex, int numPartitions) {
- JdbcState state = new JdbcState(map , partitionIndex, numPartitions, options);
+ JdbcState state = new JdbcState(map, partitionIndex, numPartitions, options);
state.prepare();
return state;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcUpdater.java b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcUpdater.java
index d2ca5b8..47c01ba 100644
--- a/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcUpdater.java
+++ b/external/storm-jdbc/src/main/java/org/apache/storm/jdbc/trident/state/JdbcUpdater.java
@@ -1,29 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.trident.state;
+import java.util.List;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseStateUpdater;
import org.apache.storm.trident.tuple.TridentTuple;
-import java.util.List;
-
-public class JdbcUpdater extends BaseStateUpdater<JdbcState> {
+public class JdbcUpdater extends BaseStateUpdater<JdbcState> {
@Override
public void updateState(JdbcState jdbcState, List<TridentTuple> tuples, TridentCollector collector) {
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcInsertBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcInsertBoltTest.java b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcInsertBoltTest.java
index 1b393e9..df0a457 100644
--- a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcInsertBoltTest.java
+++ b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcInsertBoltTest.java
@@ -1,23 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.bolt;
import com.google.common.collect.Lists;
+import java.util.HashMap;
import org.apache.storm.jdbc.common.Column;
import org.apache.storm.jdbc.common.ConnectionProvider;
import org.apache.storm.jdbc.common.HikariCPConnectionProvider;
@@ -26,8 +22,6 @@ import org.apache.storm.jdbc.mapper.SimpleJdbcMapper;
import org.junit.Assert;
import org.junit.Test;
-import java.util.HashMap;
-
/**
* Created by pbrahmbhatt on 10/29/15.
*/
@@ -45,7 +39,7 @@ public class JdbcInsertBoltTest {
bolt.withInsertQuery("test");
bolt.withTableName("test");
Assert.fail("Should have thrown IllegalArgumentException.");
- } catch(IllegalArgumentException ne) {
+ } catch (IllegalArgumentException ne) {
//expected
}
@@ -54,7 +48,7 @@ public class JdbcInsertBoltTest {
bolt.withTableName("test");
bolt.withInsertQuery("test");
Assert.fail("Should have thrown IllegalArgumentException.");
- } catch(IllegalArgumentException ne) {
+ } catch (IllegalArgumentException ne) {
//expected
}
}
@@ -63,7 +57,7 @@ public class JdbcInsertBoltTest {
try {
JdbcInsertBolt bolt = new JdbcInsertBolt(provider, mapper);
Assert.fail("Should have thrown IllegalArgumentException.");
- } catch(IllegalArgumentException ne) {
+ } catch (IllegalArgumentException ne) {
//expected
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcLookupBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcLookupBoltTest.java b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcLookupBoltTest.java
index 9a5ec09..a26fc4a 100644
--- a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcLookupBoltTest.java
+++ b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/bolt/JdbcLookupBoltTest.java
@@ -1,37 +1,28 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.bolt;
-import org.apache.storm.tuple.Fields;
import com.google.common.collect.Lists;
+import java.util.HashMap;
import org.apache.storm.jdbc.common.Column;
import org.apache.storm.jdbc.common.ConnectionProvider;
import org.apache.storm.jdbc.common.HikariCPConnectionProvider;
import org.apache.storm.jdbc.mapper.JdbcLookupMapper;
-import org.apache.storm.jdbc.mapper.JdbcMapper;
import org.apache.storm.jdbc.mapper.SimpleJdbcLookupMapper;
-import org.apache.storm.jdbc.mapper.SimpleJdbcMapper;
+import org.apache.storm.tuple.Fields;
import org.junit.Assert;
import org.junit.Test;
-import java.util.ArrayList;
-import java.util.HashMap;
-
/**
* Created by pbrahmbhatt on 10/29/15.
*/
@@ -51,7 +42,7 @@ public class JdbcLookupBoltTest {
try {
JdbcLookupBolt bolt = new JdbcLookupBolt(provider, selectQuery, mapper);
Assert.fail("Should have thrown IllegalArgumentException.");
- } catch(IllegalArgumentException ne) {
+ } catch (IllegalArgumentException ne) {
//expected
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/JdbcClientTest.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/JdbcClientTest.java b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/JdbcClientTest.java
index 05f4f31..d77e8be 100644
--- a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/JdbcClientTest.java
+++ b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/JdbcClientTest.java
@@ -1,52 +1,47 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.common;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.rules.ExpectedException;
-import org.junit.runners.model.MultipleFailureException;
-import org.junit.Test;
-
import java.sql.Connection;
-import java.sql.SQLException;
import java.sql.Timestamp;
import java.sql.Types;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runners.model.MultipleFailureException;
public class JdbcClientTest {
+ private static final String tableName = "user_details";
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
private JdbcClient client;
- private static final String tableName = "user_details";
@Before
public void setup() {
Map<String, Object> map = Maps.newHashMap();
- map.put("dataSourceClassName","org.hsqldb.jdbc.JDBCDataSource");//com.mysql.jdbc.jdbc2.optional.MysqlDataSource
+ map.put("dataSourceClassName", "org.hsqldb.jdbc.JDBCDataSource");//com.mysql.jdbc.jdbc2.optional.MysqlDataSource
map.put("dataSource.url", "jdbc:hsqldb:mem:test");//jdbc:mysql://localhost/test
- map.put("dataSource.user","SA");//root
- map.put("dataSource.password","");//password
+ map.put("dataSource.user", "SA");//root
+ map.put("dataSource.password", "");//password
ConnectionProvider connectionProvider = new HikariCPConnectionProvider(map);
connectionProvider.prepare();
@@ -64,13 +59,14 @@ public class JdbcClientTest {
List<List<Column>> rows = Lists.newArrayList(row1, row2);
client.insert(tableName, rows);
- List<List<Column>> selectedRows = client.select("select * from user_details where id = ?", Lists.newArrayList(new Column("id", 1, Types.INTEGER)));
+ List<List<Column>> selectedRows =
+ client.select("select * from user_details where id = ?", Lists.newArrayList(new Column("id", 1, Types.INTEGER)));
List<List<Column>> expectedRows = Lists.newArrayList();
expectedRows.add(row1);
Assert.assertEquals(expectedRows, selectedRows);
List<Column> row3 = createRow(3, "frank");
- List<List<Column>> moreRows = new ArrayList<List<Column>>();
+ List<List<Column>> moreRows = new ArrayList<List<Column>>();
moreRows.add(row3);
client.executeInsertQuery("insert into user_details values(?,?,?)", moreRows);
@@ -85,9 +81,6 @@ public class JdbcClientTest {
Assert.assertEquals(rows, selectedRows);
}
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
@Test
public void testInsertConnectionError() {
@@ -95,9 +88,9 @@ public class JdbcClientTest {
this.client = new JdbcClient(connectionProvider, 60);
List<Column> row = createRow(1, "frank");
- List<List<Column>> rows = new ArrayList<List<Column>>();
+ List<List<Column>> rows = new ArrayList<List<Column>>();
rows.add(row);
- String query = "insert into user_details values(?,?,?)";
+ String query = "insert into user_details values(?,?,?)";
thrown.expect(MultipleFailureException.class);
client.executeInsertQuery(query, rows);
@@ -105,9 +98,9 @@ public class JdbcClientTest {
private List<Column> createRow(int id, String name) {
return Lists.newArrayList(
- new Column("ID", id, Types.INTEGER),
- new Column("USER_NAME", name, Types.VARCHAR),
- new Column("CREATED_TIMESTAMP", new Timestamp(System.currentTimeMillis()) , Types.TIMESTAMP));
+ new Column("ID", id, Types.INTEGER),
+ new Column("USER_NAME", name, Types.VARCHAR),
+ new Column("CREATED_TIMESTAMP", new Timestamp(System.currentTimeMillis()), Types.TIMESTAMP));
}
@After
http://git-wip-us.apache.org/repos/asf/storm/blob/6d20c4af/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/UtilTest.java
----------------------------------------------------------------------
diff --git a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/UtilTest.java b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/UtilTest.java
index 8b6e8b3..76d5f6a 100644
--- a/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/UtilTest.java
+++ b/external/storm-jdbc/src/test/java/org/apache/storm/jdbc/common/UtilTest.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.jdbc.common;
import java.sql.Date;
@@ -23,7 +18,8 @@ import java.sql.Timestamp;
import java.sql.Types;
import org.junit.Test;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
public class UtilTest {
[26/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-hdfs
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java
index 67ccde6..39a151f 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java
@@ -1,32 +1,27 @@
-
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.spout;
-import org.apache.storm.Config;
-import org.apache.storm.spout.SpoutOutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.storm.hdfs.common.HdfsUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -34,46 +29,41 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
-import org.junit.Before;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.storm.Config;
+import org.apache.storm.hdfs.common.HdfsUtils;
+import org.apache.storm.hdfs.common.HdfsUtils.Pair;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.apache.storm.spout.SpoutOutputCollector;
+import org.apache.storm.task.TopologyContext;
import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.storm.hdfs.common.HdfsUtils.Pair;
-import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
-import org.junit.ClassRule;
-
public class TestHdfsSpout {
+ private static final Configuration conf = new Configuration();
@ClassRule
public static MiniDFSClusterRule DFS_CLUSTER_RULE = new MiniDFSClusterRule();
+ private static DistributedFileSystem fs;
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();
public File baseFolder;
-
private Path source;
private Path archive;
private Path badfiles;
- private static DistributedFileSystem fs;
- private static final Configuration conf = new Configuration();
-
@BeforeClass
public static void setupClass() throws IOException {
fs = DFS_CLUSTER_RULE.getDfscluster().getFileSystem();
@@ -84,6 +74,50 @@ public class TestHdfsSpout {
fs.close();
}
+ private static <T> T getField(HdfsSpout spout, String fieldName) throws NoSuchFieldException, IllegalAccessException {
+ Field readerFld = HdfsSpout.class.getDeclaredField(fieldName);
+ readerFld.setAccessible(true);
+ return (T) readerFld.get(spout);
+ }
+
+ private static boolean getBoolField(HdfsSpout spout, String fieldName) throws NoSuchFieldException, IllegalAccessException {
+ Field readerFld = HdfsSpout.class.getDeclaredField(fieldName);
+ readerFld.setAccessible(true);
+ return readerFld.getBoolean(spout);
+ }
+
+ private static List<String> readTextFile(FileSystem fs, String f) throws IOException {
+ Path file = new Path(f);
+ FSDataInputStream x = fs.open(file);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(x));
+ String line = null;
+ ArrayList<String> result = new ArrayList<>();
+ while ((line = reader.readLine()) != null) {
+ result.add(line);
+ }
+ return result;
+ }
+
+ private static void createSeqFile(FileSystem fs, Path file, int rowCount) throws IOException {
+
+ Configuration conf = new Configuration();
+ try {
+ if (fs.exists(file)) {
+ fs.delete(file, false);
+ }
+
+ SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, file, IntWritable.class, Text.class);
+ for (int i = 0; i < rowCount; i++) {
+ w.append(new IntWritable(i), new Text("line " + i));
+ }
+ w.close();
+ System.out.println("done");
+ } catch (IOException e) {
+ e.printStackTrace();
+
+ }
+ }
+
@Before
public void setup() throws Exception {
baseFolder = tempFolder.newFolder("hdfsspout");
@@ -395,18 +429,6 @@ public class TestHdfsSpout {
}
}
- private static <T> T getField(HdfsSpout spout, String fieldName) throws NoSuchFieldException, IllegalAccessException {
- Field readerFld = HdfsSpout.class.getDeclaredField(fieldName);
- readerFld.setAccessible(true);
- return (T) readerFld.get(spout);
- }
-
- private static boolean getBoolField(HdfsSpout spout, String fieldName) throws NoSuchFieldException, IllegalAccessException {
- Field readerFld = HdfsSpout.class.getDeclaredField(fieldName);
- readerFld.setAccessible(true);
- return readerFld.getBoolean(spout);
- }
-
@Test
public void testSimpleSequenceFile() throws Exception {
//1) create a couple files to consume
@@ -450,13 +472,14 @@ public class TestHdfsSpout {
Assert.assertEquals(2, listDir(source).size());
// 2) run spout
- try (AutoCloseableHdfsSpout closeableSpout = makeSpout(MockTextFailingReader.class.getName(), MockTextFailingReader.defaultFields)) {
+ try (
+ AutoCloseableHdfsSpout closeableSpout = makeSpout(MockTextFailingReader.class.getName(), MockTextFailingReader.defaultFields)) {
HdfsSpout spout = closeableSpout.spout;
Map<String, Object> conf = getCommonConfigs();
openSpout(spout, 0, conf);
List<String> res = runSpout(spout, "r11");
- String[] expected = new String[]{"[line 0]", "[line 1]", "[line 2]", "[line 0]", "[line 1]", "[line 2]"};
+ String[] expected = new String[]{ "[line 0]", "[line 1]", "[line 2]", "[line 0]", "[line 1]", "[line 2]" };
Assert.assertArrayEquals(expected, res.toArray());
// 3) make sure 6 lines (3 from each file) were read in all
@@ -576,18 +599,6 @@ public class TestHdfsSpout {
}
}
- private static List<String> readTextFile(FileSystem fs, String f) throws IOException {
- Path file = new Path(f);
- FSDataInputStream x = fs.open(file);
- BufferedReader reader = new BufferedReader(new InputStreamReader(x));
- String line = null;
- ArrayList<String> result = new ArrayList<>();
- while ((line = reader.readLine()) != null) {
- result.add(line);
- }
- return result;
- }
-
private Map<String, Object> getCommonConfigs() {
Map<String, Object> topoConf = new HashMap<>();
topoConf.put(Config.TOPOLOGY_ACKER_EXECUTORS, "0");
@@ -596,29 +607,15 @@ public class TestHdfsSpout {
private AutoCloseableHdfsSpout makeSpout(String readerType, String[] outputFields) {
HdfsSpout spout = new HdfsSpout().withOutputFields(outputFields)
- .setReaderType(readerType)
- .setHdfsUri(DFS_CLUSTER_RULE.getDfscluster().getURI().toString())
- .setSourceDir(source.toString())
- .setArchiveDir(archive.toString())
- .setBadFilesDir(badfiles.toString());
+ .setReaderType(readerType)
+ .setHdfsUri(DFS_CLUSTER_RULE.getDfscluster().getURI().toString())
+ .setSourceDir(source.toString())
+ .setArchiveDir(archive.toString())
+ .setBadFilesDir(badfiles.toString());
return new AutoCloseableHdfsSpout(spout);
}
- private static class AutoCloseableHdfsSpout implements AutoCloseable {
-
- private final HdfsSpout spout;
-
- public AutoCloseableHdfsSpout(HdfsSpout spout) {
- this.spout = spout;
- }
-
- @Override
- public void close() throws Exception {
- spout.close();
- }
- }
-
private void openSpout(HdfsSpout spout, int spoutId, Map<String, Object> topoConf) {
MockCollector collector = new MockCollector();
spout.open(topoConf, new MockTopologyContext(spoutId, topoConf), collector);
@@ -665,23 +662,17 @@ public class TestHdfsSpout {
os.close();
}
- private static void createSeqFile(FileSystem fs, Path file, int rowCount) throws IOException {
+ private static class AutoCloseableHdfsSpout implements AutoCloseable {
- Configuration conf = new Configuration();
- try {
- if (fs.exists(file)) {
- fs.delete(file, false);
- }
+ private final HdfsSpout spout;
- SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, file, IntWritable.class, Text.class);
- for (int i = 0; i < rowCount; i++) {
- w.append(new IntWritable(i), new Text("line " + i));
- }
- w.close();
- System.out.println("done");
- } catch (IOException e) {
- e.printStackTrace();
+ public AutoCloseableHdfsSpout(HdfsSpout spout) {
+ this.spout = spout;
+ }
+ @Override
+ public void close() throws Exception {
+ spout.close();
}
}
@@ -729,7 +720,7 @@ public class TestHdfsSpout {
// throws ParseException. Effectively produces 3 lines (1,2 & 3) from each file read
static class MockTextFailingReader extends TextFileReader {
- public static final String[] defaultFields = {"line"};
+ public static final String[] defaultFields = { "line" };
int readAttempts = 0;
public MockTextFailingReader(FileSystem fs, Path file, Map<String, Object> conf) throws IOException {
@@ -753,7 +744,11 @@ public class TestHdfsSpout {
private final int componentId;
public MockTopologyContext(int componentId, Map<String, Object> topoConf) {
- // StormTopology topology, Map<String, Object> topoConf, Map<Integer, String> taskToComponent, Map<String, List<Integer>> componentToSortedTasks, Map<String, Map<String, Fields>> componentToStreamToFields, String stormId, String codeDir, String pidDir, Integer taskId, Integer workerPort, List<Integer> workerTasks, Map<String, Object> defaultResources, Map<String, Object> userResources, Map<String, Object> executorData, Map<Integer, Map<Integer, Map<String, IMetric>>> registeredMetrics, Atom openOrPrepareWasCalled
+ // StormTopology topology, Map<String, Object> topoConf, Map<Integer, String> taskToComponent, Map<String, List<Integer>>
+ // componentToSortedTasks, Map<String, Map<String, Fields>> componentToStreamToFields, String stormId, String codeDir, String
+ // pidDir, Integer taskId, Integer workerPort, List<Integer> workerTasks, Map<String, Object> defaultResources, Map<String,
+ // Object> userResources, Map<String, Object> executorData, Map<Integer, Map<Integer, Map<String, IMetric>>>
+ // registeredMetrics, Atom openOrPrepareWasCalled
super(null, topoConf, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);
this.componentId = componentId;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java
index c631838..760e099 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java
@@ -1,12 +1,7 @@
-
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
@@ -17,6 +12,8 @@
package org.apache.storm.hdfs.spout;
+import java.io.File;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -27,17 +24,13 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import java.io.File;
-import java.io.IOException;
-
public class TestProgressTracker {
- private FileSystem fs;
- private Configuration conf = new Configuration();
-
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();
public File baseFolder;
+ private FileSystem fs;
+ private Configuration conf = new Configuration();
@Before
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterRule.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterRule.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterRule.java
index 1d1f6c7..b94fb53 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterRule.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterRule.java
@@ -26,7 +26,7 @@ import org.junit.runners.model.Statement;
public class MiniDFSClusterRule implements TestRule {
private static final String TEST_BUILD_DATA = "test.build.data";
-
+
private final Supplier<Configuration> hadoopConfSupplier;
private Configuration hadoopConf;
private MiniDFSCluster dfscluster;
@@ -34,11 +34,11 @@ public class MiniDFSClusterRule implements TestRule {
public MiniDFSClusterRule() {
this(() -> new Configuration());
}
-
+
public MiniDFSClusterRule(Supplier<Configuration> hadoopConfSupplier) {
this.hadoopConfSupplier = hadoopConfSupplier;
}
-
+
public Configuration getHadoopConf() {
return hadoopConf;
}
@@ -46,7 +46,7 @@ public class MiniDFSClusterRule implements TestRule {
public MiniDFSCluster getDfscluster() {
return dfscluster;
}
-
+
@Override
public Statement apply(Statement base, Description description) {
return new Statement() {
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/HdfsStateTest.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/HdfsStateTest.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/HdfsStateTest.java
index e018016..3d7218c 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/HdfsStateTest.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/HdfsStateTest.java
@@ -1,34 +1,16 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.hdfs.trident;
-import org.apache.storm.Config;
-import org.apache.storm.tuple.Fields;
-import org.apache.commons.io.FileUtils;
-import org.apache.storm.hdfs.trident.format.DelimitedRecordFormat;
-import org.apache.storm.hdfs.trident.format.FileNameFormat;
-import org.apache.storm.hdfs.trident.format.RecordFormat;
-import org.apache.storm.hdfs.trident.rotation.FileRotationPolicy;
-import org.apache.storm.hdfs.trident.rotation.FileSizeRotationPolicy;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.apache.storm.trident.tuple.TridentTuple;
+package org.apache.storm.hdfs.trident;
import java.io.File;
import java.io.IOException;
@@ -41,6 +23,18 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.commons.io.FileUtils;
+import org.apache.storm.Config;
+import org.apache.storm.hdfs.trident.format.DelimitedRecordFormat;
+import org.apache.storm.hdfs.trident.format.FileNameFormat;
+import org.apache.storm.hdfs.trident.format.RecordFormat;
+import org.apache.storm.hdfs.trident.rotation.FileRotationPolicy;
+import org.apache.storm.hdfs.trident.rotation.FileSizeRotationPolicy;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Fields;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
@@ -56,30 +50,6 @@ public class HdfsStateTest {
private static final String INDEX_FILE_PREFIX = ".index.";
private final TestFileNameFormat fileNameFormat = new TestFileNameFormat();
- private static class TestFileNameFormat implements FileNameFormat {
- private String currentFileName = "";
-
- @Override
- public void prepare(Map<String, Object> conf, int partitionIndex, int numPartitions) {
-
- }
-
- @Override
- public String getName(long rotation, long timeStamp) {
- currentFileName = FILE_NAME_PREFIX + Long.toString(rotation);
- return currentFileName;
- }
-
- @Override
- public String getPath() {
- return TEST_OUT_DIR;
- }
-
- public String getCurrentFileName() {
- return currentFileName;
- }
- }
-
private HdfsState createHdfsState() {
Fields hdfsFields = new Fields("f1");
@@ -89,10 +59,10 @@ public class HdfsStateTest {
FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
HdfsState.Options options = new HdfsState.HdfsFileOptions()
- .withFileNameFormat(fileNameFormat)
- .withRecordFormat(recordFormat)
- .withRotationPolicy(rotationPolicy)
- .withFsUrl("file://" + TEST_OUT_DIR);
+ .withFileNameFormat(fileNameFormat)
+ .withRecordFormat(recordFormat)
+ .withRotationPolicy(rotationPolicy)
+ .withFsUrl("file://" + TEST_OUT_DIR);
Map<String, Object> conf = new HashMap<>();
conf.put(Config.TOPOLOGY_NAME, TEST_TOPOLOGY_NAME);
@@ -123,7 +93,6 @@ public class HdfsStateTest {
FileUtils.deleteQuietly(new File(TEST_OUT_DIR));
}
-
@Test
public void testPrepare() throws Exception {
HdfsState state = createHdfsState();
@@ -212,7 +181,7 @@ public class HdfsStateTest {
/*
* total tuples should be
* recovered (batch-1 + batch-2) + replayed (batch-3)
- */
+ */
List<String> lines = getLinesFromCurrentDataFile();
int preReplayCount = batch1Count + batch2Count + batch3Count;
int expectedTupleCount = batch1Count + batch2Count + batch3ReplayCount;
@@ -220,4 +189,28 @@ public class HdfsStateTest {
Assert.assertNotEquals(preReplayCount, lines.size());
Assert.assertEquals(expectedTupleCount, lines.size());
}
+
+ private static class TestFileNameFormat implements FileNameFormat {
+ private String currentFileName = "";
+
+ @Override
+ public void prepare(Map<String, Object> conf, int partitionIndex, int numPartitions) {
+
+ }
+
+ @Override
+ public String getName(long rotation, long timeStamp) {
+ currentFileName = FILE_NAME_PREFIX + Long.toString(rotation);
+ return currentFileName;
+ }
+
+ @Override
+ public String getPath() {
+ return TEST_OUT_DIR;
+ }
+
+ public String getCurrentFileName() {
+ return currentFileName;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/7da98cf0/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/format/TestSimpleFileNameFormat.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/format/TestSimpleFileNameFormat.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/format/TestSimpleFileNameFormat.java
index 232b4cf..d4840b0 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/format/TestSimpleFileNameFormat.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/trident/format/TestSimpleFileNameFormat.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hdfs.trident.format;
import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
-
import org.apache.storm.utils.Utils;
import org.junit.Assert;
import org.junit.Test;
@@ -36,7 +30,7 @@ public class TestSimpleFileNameFormat {
Assert.assertEquals("/storm", path);
String time = new SimpleDateFormat("yyyyMMddHHmmss").format(now);
- Assert.assertEquals(time+".1.txt", name);
+ Assert.assertEquals(time + ".1.txt", name);
}
@Test
@@ -49,7 +43,7 @@ public class TestSimpleFileNameFormat {
long now = System.currentTimeMillis();
String path = format.getPath();
String name = format.getName(1, now);
-
+
Assert.assertEquals("/mypath", path);
String time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(now);
String host = null;
@@ -58,13 +52,13 @@ public class TestSimpleFileNameFormat {
} catch (UnknownHostException e) {
e.printStackTrace();
}
- Assert.assertEquals(time+"."+host+".3.1.txt", name);
+ Assert.assertEquals(time + "." + host + ".3.1.txt", name);
}
- @Test(expected=IllegalArgumentException.class)
+ @Test(expected = IllegalArgumentException.class)
public void testTimeFormat() {
SimpleFileNameFormat format = new SimpleFileNameFormat()
- .withTimeFormat("xyz");
+ .withTimeFormat("xyz");
format.prepare(null, 3, 5);
}
}
[11/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbStore.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbStore.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbStore.java
index 2e7ed02..94f2fed 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbStore.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbStore.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore.rocksdb;
@@ -25,7 +18,6 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicReference;
-
import org.apache.storm.DaemonConfig;
import org.apache.storm.metric.StormMetricsRegistry;
import org.apache.storm.metricstore.AggLevel;
@@ -48,9 +40,9 @@ import org.slf4j.LoggerFactory;
public class RocksDbStore implements MetricStore, AutoCloseable {
+ static final int INVALID_METADATA_STRING_ID = 0;
private static final Logger LOG = LoggerFactory.getLogger(RocksDbStore.class);
private static final int MAX_QUEUE_CAPACITY = 4000;
- static final int INVALID_METADATA_STRING_ID = 0;
RocksDB db;
private ReadOnlyStringMetadataCache readOnlyStringMetadataCache = null;
private BlockingQueue queue = new LinkedBlockingQueue(MAX_QUEUE_CAPACITY);
@@ -58,10 +50,6 @@ public class RocksDbStore implements MetricStore, AutoCloseable {
private MetricsCleaner metricsCleaner = null;
private Meter failureMeter = null;
- interface RocksDbScanCallback {
- boolean cb(RocksDbKey key, RocksDbValue val); // return false to stop scan
- }
-
/**
* Create metric store instance using the configurations provided via the config map.
*
@@ -132,7 +120,7 @@ public class RocksDbStore implements MetricStore, AutoCloseable {
if (!(config.containsKey(DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING))) {
throw new MetricException("Not a vaild RocksDB configuration - Does not specify creation policy "
- + DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING);
+ + DaemonConfig.STORM_ROCKSDB_CREATE_IF_MISSING);
}
// validate path defined
@@ -147,17 +135,17 @@ public class RocksDbStore implements MetricStore, AutoCloseable {
if (!(config.containsKey(DaemonConfig.STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY))) {
throw new MetricException("Not a valid RocksDB configuration - Missing metadata string cache size "
- + DaemonConfig.STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY);
+ + DaemonConfig.STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY);
}
if (!config.containsKey(DaemonConfig.STORM_ROCKSDB_METRIC_RETENTION_HOURS)) {
throw new MetricException("Not a valid RocksDB configuration - Missing metric retention "
- + DaemonConfig.STORM_ROCKSDB_METRIC_RETENTION_HOURS);
+ + DaemonConfig.STORM_ROCKSDB_METRIC_RETENTION_HOURS);
}
}
private String getRocksDbAbsoluteDir(Map<String, Object> conf) throws MetricException {
- String storePath = (String)conf.get(DaemonConfig.STORM_ROCKSDB_LOCATION);
+ String storePath = (String) conf.get(DaemonConfig.STORM_ROCKSDB_LOCATION);
if (storePath == null) {
throw new MetricException("Not a vaild RocksDB configuration - Missing store location " + DaemonConfig.STORM_ROCKSDB_LOCATION);
} else {
@@ -201,7 +189,7 @@ public class RocksDbStore implements MetricStore, AutoCloseable {
* Fill out the numeric values for a metric.
*
* @param metric Metric to populate
- * @return true if the metric was populated, false otherwise
+ * @return true if the metric was populated, false otherwise
* @throws MetricException if read from database fails
*/
@Override
@@ -234,7 +222,7 @@ public class RocksDbStore implements MetricStore, AutoCloseable {
}
RocksDbKey key = RocksDbKey.createMetricKey(metric.getAggLevel(), topologyId, metric.getTimestamp(), metricId,
- componentId, executorId, hostId, metric.getPort(), streamId);
+ componentId, executorId, hostId, metric.getPort(), streamId);
return populateFromKey(key, metric);
}
@@ -461,9 +449,9 @@ public class RocksDbStore implements MetricStore, AutoCloseable {
for (AggLevel aggLevel : filter.getAggLevels()) {
RocksDbKey startKey = RocksDbKey.createMetricKey(aggLevel, startTopologyId, startTime, startMetricId,
- startComponentId, startExecutorId, startHostId, startPort, startStreamId);
+ startComponentId, startExecutorId, startHostId, startPort, startStreamId);
RocksDbKey endKey = RocksDbKey.createMetricKey(aggLevel, endTopologyId, endTime, endMetricId,
- endComponentId, endExecutorId, endHostId, endPort, endStreamId);
+ endComponentId, endExecutorId, endHostId, endPort, endStreamId);
RocksIterator iterator = db.newIterator(ro);
for (iterator.seek(startKey.getRaw()); iterator.isValid(); iterator.next()) {
@@ -519,7 +507,7 @@ public class RocksDbStore implements MetricStore, AutoCloseable {
String streamId = metadataIdToString(KeyType.STREAM_ID_STRING, key.getStreamId(), idToStringCache);
Metric metric = new Metric(metricName, timestamp, topologyId, 0.0, componentId, executorId, hostname,
- streamId, key.getPort(), aggLevel);
+ streamId, key.getPort(), aggLevel);
val.populateMetric(metric);
@@ -561,7 +549,7 @@ public class RocksDbStore implements MetricStore, AutoCloseable {
s = rdbValue.getMetdataString();
lookupCache.put(id, s);
return s;
- } catch (RocksDBException e) {
+ } catch (RocksDBException e) {
if (this.failureMeter != null) {
this.failureMeter.mark();
}
@@ -635,5 +623,9 @@ public class RocksDbStore implements MetricStore, AutoCloseable {
}
}
}
+
+ interface RocksDbScanCallback {
+ boolean cb(RocksDbKey key, RocksDbValue val); // return false to stop scan
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbValue.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbValue.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbValue.java
index 58b2c76..0da7b92 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbValue.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/RocksDbValue.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore.rocksdb;
@@ -50,11 +43,11 @@ import org.apache.storm.metricstore.Metric;
*/
class RocksDbValue {
- private static int METRIC_VALUE_SIZE = 41;
- private byte[] value;
private static final byte CURRENT_METADATA_VERSION = 0;
private static final byte CURRENT_METRIC_VERSION = 0;
+ private static int METRIC_VALUE_SIZE = 41;
private static int MIN_METADATA_VALUE_SIZE = 9;
+ private byte[] value;
/**
* Constructor from raw data.
@@ -98,7 +91,7 @@ class RocksDbValue {
/**
* Get the metadata string portion of the value. Assumes the value is metadata.
*
- * @return the metadata string
+ * @return the metadata string
*/
String getMetdataString() {
if (this.value.length < MIN_METADATA_VALUE_SIZE) {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadata.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadata.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadata.java
index 6f54a58..d890582 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadata.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadata.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore.rocksdb;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadataCache.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadataCache.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadataCache.java
index 7ce8435..d70076f 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadataCache.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/StringMetadataCache.java
@@ -1,19 +1,12 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore.rocksdb;
@@ -39,12 +32,23 @@ import org.slf4j.LoggerFactory;
*/
public class StringMetadataCache implements LruMap.CacheEvictionCallback<String, StringMetadata>,
- WritableStringMetadataCache, ReadOnlyStringMetadataCache {
+ WritableStringMetadataCache, ReadOnlyStringMetadataCache {
private static final Logger LOG = LoggerFactory.getLogger(StringMetadataCache.class);
+ private static StringMetadataCache instance = null;
private Map<String, StringMetadata> lruStringCache;
private Map<Integer, String> hashToString = new ConcurrentHashMap<>();
private RocksDbMetricsWriter dbWriter;
- private static StringMetadataCache instance = null;
+
+ /**
+ * Constructor to create a cache.
+ *
+ * @param dbWriter The rocks db writer instance the cache should use when evicting data
+ * @param capacity The cache size
+ */
+ private StringMetadataCache(RocksDbMetricsWriter dbWriter, int capacity) {
+ lruStringCache = Collections.synchronizedMap(new LruMap<>(capacity, this));
+ this.dbWriter = dbWriter;
+ }
/**
* Initializes the cache instance.
@@ -87,22 +91,15 @@ public class StringMetadataCache implements LruMap.CacheEvictionCallback<String,
}
}
- /**
- * Constructor to create a cache.
- *
- * @param dbWriter The rocks db writer instance the cache should use when evicting data
- * @param capacity The cache size
- */
- private StringMetadataCache(RocksDbMetricsWriter dbWriter, int capacity) {
- lruStringCache = Collections.synchronizedMap(new LruMap<>(capacity, this));
- this.dbWriter = dbWriter;
+ static void cleanUp() {
+ instance = null;
}
/**
* Get the string metadata from the cache.
*
* @param s The string to look for
- * @return the metadata associated with the string or null if not found
+ * @return the metadata associated with the string or null if not found
*/
public StringMetadata get(String s) {
return lruStringCache.get(s);
@@ -169,7 +166,7 @@ public class StringMetadataCache implements LruMap.CacheEvictionCallback<String,
* Determines if a string Id is contained in the cache.
*
* @param stringId The string Id to check
- * @return true if the Id is in the cache, false otherwise
+ * @return true if the Id is in the cache, false otherwise
*/
public boolean contains(Integer stringId) {
return hashToString.containsKey(stringId);
@@ -179,7 +176,7 @@ public class StringMetadataCache implements LruMap.CacheEvictionCallback<String,
* Returns the string matching the string Id if in the cache.
*
* @param stringId The string Id to check
- * @return the associated string if the Id is in the cache, null otherwise
+ * @return the associated string if the Id is in the cache, null otherwise
*/
public String getMetadataString(Integer stringId) {
return hashToString.get(stringId);
@@ -188,15 +185,11 @@ public class StringMetadataCache implements LruMap.CacheEvictionCallback<String,
/**
* Get the map of the cache contents. Provided to allow writing the data to RocksDB on shutdown.
*
- * @return the string metadata map entrySet
+ * @return the string metadata map entrySet
*/
public Set<Map.Entry<String, StringMetadata>> entrySet() {
return lruStringCache.entrySet();
}
- static void cleanUp() {
- instance = null;
- }
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/WritableStringMetadataCache.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/WritableStringMetadataCache.java b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/WritableStringMetadataCache.java
index 2d4165f..ab47fdf 100644
--- a/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/WritableStringMetadataCache.java
+++ b/storm-server/src/main/java/org/apache/storm/metricstore/rocksdb/WritableStringMetadataCache.java
@@ -1,26 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metricstore.rocksdb;
import java.util.Map;
import java.util.Set;
-
import org.apache.http.annotation.NotThreadSafe;
import org.apache.storm.metricstore.MetricException;
@@ -48,7 +40,7 @@ public interface WritableStringMetadataCache extends ReadOnlyStringMetadataCache
/**
* Get the map of the cache contents. Provided to allow writing the data to RocksDB on shutdown.
*
- * @return the string metadata map entrySet
+ * @return the string metadata map entrySet
*/
Set<Map.Entry<String, StringMetadata>> entrySet();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/nimbus/AssignmentDistributionService.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/AssignmentDistributionService.java b/storm-server/src/main/java/org/apache/storm/nimbus/AssignmentDistributionService.java
index 558e570..b912513 100644
--- a/storm-server/src/main/java/org/apache/storm/nimbus/AssignmentDistributionService.java
+++ b/storm-server/src/main/java/org/apache/storm/nimbus/AssignmentDistributionService.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.nimbus;
@@ -27,11 +21,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
-
import org.apache.storm.DaemonConfig;
import org.apache.storm.daemon.supervisor.Supervisor;
import org.apache.storm.generated.SupervisorAssignments;
-import org.apache.storm.scheduler.SupervisorDetails;
import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.SupervisorClient;
@@ -95,6 +87,17 @@ public class AssignmentDistributionService implements Closeable {
private boolean isLocalMode = false; // boolean cache for local mode decision
/**
+ * Factory method for initialize a instance.
+ * @param conf config.
+ * @return an instance of {@link AssignmentDistributionService}
+ */
+ public static AssignmentDistributionService getInstance(Map conf) {
+ AssignmentDistributionService service = new AssignmentDistributionService();
+ service.prepare(conf);
+ return service;
+ }
+
+ /**
* Function for initialization.
*
* @param conf config
@@ -163,6 +166,47 @@ public class AssignmentDistributionService implements Closeable {
}
}
+ public void addLocalSupervisor(Supervisor supervisor) {
+ this.localSupervisors.put(supervisor.getId(), supervisor);
+ }
+
+ private Integer nextQueueId() {
+ return this.random.nextInt(threadsNum);
+ }
+
+ private LinkedBlockingQueue<NodeAssignments> nextQueue() {
+ return this.assignmentsQueue.get(nextQueueId());
+ }
+
+ private LinkedBlockingQueue<NodeAssignments> getQueueById(Integer queueIndex) {
+ return this.assignmentsQueue.get(queueIndex);
+ }
+
+ /**
+ * Get an assignments from the target queue with the specific index.
+ * @param queueIndex index of the queue
+ * @return an {@link NodeAssignments}
+ * @throws InterruptedException
+ */
+ public NodeAssignments nextAssignments(Integer queueIndex) throws InterruptedException {
+ NodeAssignments target = null;
+ while (true) {
+ target = getQueueById(queueIndex).poll();
+ if (target != null) {
+ return target;
+ }
+ Time.sleep(100L);
+ }
+ }
+
+ public boolean isActive() {
+ return this.active;
+ }
+
+ public Map getConf() {
+ return this.conf;
+ }
+
static class NodeAssignments {
private String node;
private String host;
@@ -177,7 +221,7 @@ public class AssignmentDistributionService implements Closeable {
}
public static NodeAssignments getInstance(String node, String host, Integer serverPort,
- SupervisorAssignments assignments) {
+ SupervisorAssignments assignments) {
return new NodeAssignments(node, host, serverPort, assignments);
}
@@ -237,12 +281,12 @@ public class AssignmentDistributionService implements Closeable {
supervisor.sendSupervisorAssignments(assignments.getAssignments());
} else {
LOG.error("Can not find node {} for assignments distribution", assignments.getNode());
- throw new RuntimeException("null for node " + assignments.getNode() + " supervisor instance.");
+ throw new RuntimeException("null for node " + assignments.getNode() + " supervisor instance.");
}
} else {
// distributed mode
try (SupervisorClient client = SupervisorClient.getConfiguredClient(service.getConf(),
- assignments.getHost(), assignments.getServerPort())){
+ assignments.getHost(), assignments.getServerPort())) {
try {
client.getClient().sendSupervisorAssignments(assignments.getAssignments());
} catch (Exception e) {
@@ -257,56 +301,4 @@ public class AssignmentDistributionService implements Closeable {
}
}
}
-
- public void addLocalSupervisor(Supervisor supervisor) {
- this.localSupervisors.put(supervisor.getId(), supervisor);
- }
-
- private Integer nextQueueId() {
- return this.random.nextInt(threadsNum);
- }
-
- private LinkedBlockingQueue<NodeAssignments> nextQueue() {
- return this.assignmentsQueue.get(nextQueueId());
- }
-
- private LinkedBlockingQueue<NodeAssignments> getQueueById(Integer queueIndex) {
- return this.assignmentsQueue.get(queueIndex);
- }
-
- /**
- * Get an assignments from the target queue with the specific index.
- * @param queueIndex index of the queue
- * @return an {@link NodeAssignments}
- * @throws InterruptedException
- */
- public NodeAssignments nextAssignments(Integer queueIndex) throws InterruptedException {
- NodeAssignments target = null;
- while (true) {
- target = getQueueById(queueIndex).poll();
- if (target != null) {
- return target;
- }
- Time.sleep(100L);
- }
- }
-
- public boolean isActive() {
- return this.active;
- }
-
- public Map getConf() {
- return this.conf;
- }
-
- /**
- * Factory method for initialize a instance.
- * @param conf config.
- * @return an instance of {@link AssignmentDistributionService}
- */
- public static AssignmentDistributionService getInstance(Map conf) {
- AssignmentDistributionService service = new AssignmentDistributionService();
- service.prepare(conf);
- return service;
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/nimbus/DefaultTopologyValidator.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/DefaultTopologyValidator.java b/storm-server/src/main/java/org/apache/storm/nimbus/DefaultTopologyValidator.java
index ab164ca..7fef7bf 100644
--- a/storm-server/src/main/java/org/apache/storm/nimbus/DefaultTopologyValidator.java
+++ b/storm-server/src/main/java/org/apache/storm/nimbus/DefaultTopologyValidator.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.nimbus;
@@ -30,7 +24,7 @@ public class DefaultTopologyValidator implements ITopologyValidator {
private static final Logger LOG = LoggerFactory.getLogger(DefaultTopologyValidator.class);
@Override
- public void prepare(Map<String, Object> StormConf){
+ public void prepare(Map<String, Object> StormConf) {
}
@Override
@@ -63,5 +57,5 @@ public class DefaultTopologyValidator implements ITopologyValidator {
}
}
}
- }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/nimbus/ILeaderElector.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/ILeaderElector.java b/storm-server/src/main/java/org/apache/storm/nimbus/ILeaderElector.java
index e362fbc..05cdbc1 100644
--- a/storm-server/src/main/java/org/apache/storm/nimbus/ILeaderElector.java
+++ b/storm-server/src/main/java/org/apache/storm/nimbus/ILeaderElector.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.nimbus;
import java.io.Closeable;
@@ -62,7 +57,7 @@ public interface ILeaderElector extends Closeable {
* Get list of current nimbus addresses.
* @return list of current nimbus addresses, includes leader.
*/
- List<NimbusInfo> getAllNimbuses()throws Exception;
+ List<NimbusInfo> getAllNimbuses() throws Exception;
/**
* Method called to allow for cleanup. once close this object can not be reused.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyActionNotifierPlugin.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyActionNotifierPlugin.java b/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyActionNotifierPlugin.java
index f36a4a8..8247a85 100644
--- a/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyActionNotifierPlugin.java
+++ b/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyActionNotifierPlugin.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.nimbus;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyValidator.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyValidator.java b/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyValidator.java
index 46c4073..88d756d 100644
--- a/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyValidator.java
+++ b/storm-server/src/main/java/org/apache/storm/nimbus/ITopologyValidator.java
@@ -1,30 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.nimbus;
+import java.util.Map;
import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.generated.StormTopology;
-import java.util.Map;
public interface ITopologyValidator {
void prepare(Map<String, Object> StormConf);
void validate(String topologyName, Map<String, Object> topologyConf, StormTopology topology)
- throws InvalidTopologyException;
+ throws InvalidTopologyException;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/nimbus/IWorkerHeartbeatsRecoveryStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/IWorkerHeartbeatsRecoveryStrategy.java b/storm-server/src/main/java/org/apache/storm/nimbus/IWorkerHeartbeatsRecoveryStrategy.java
index bde58dc..8457de8 100644
--- a/storm-server/src/main/java/org/apache/storm/nimbus/IWorkerHeartbeatsRecoveryStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/nimbus/IWorkerHeartbeatsRecoveryStrategy.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.nimbus;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java b/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java
index 59cd462..68a2391 100644
--- a/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java
+++ b/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java
@@ -1,34 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.nimbus;
import com.google.common.base.Joiner;
import com.google.common.collect.Sets;
-
-import javax.security.auth.Subject;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-
+import javax.security.auth.Subject;
import org.apache.commons.io.IOUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.leader.LeaderLatch;
@@ -54,21 +46,17 @@ import org.slf4j.LoggerFactory;
*/
public class LeaderListenerCallback {
private static final Logger LOG = LoggerFactory.getLogger(LeaderListenerCallback.class);
-
+ private static final String STORM_JAR_SUFFIX = "-stormjar.jar";
+ private static final String STORM_CODE_SUFFIX = "-stormcode.ser";
+ private static final String STORM_CONF_SUFFIX = "-stormconf.ser";
private final BlobStore blobStore;
private final TopoCache tc;
private final IStormClusterState clusterState;
-
private final CuratorFramework zk;
private final LeaderLatch leaderLatch;
-
private final Map conf;
private final List<ACL> acls;
- private static final String STORM_JAR_SUFFIX = "-stormjar.jar";
- private static final String STORM_CODE_SUFFIX = "-stormcode.ser";
- private static final String STORM_CONF_SUFFIX = "-stormconf.ser";
-
/**
* Constructor for {@LeaderListenerCallback}.
* @param conf config
@@ -103,7 +91,7 @@ public class LeaderListenerCallback {
clusterState.setAssignmentsBackendSynchronized();
Set<String> activeTopologyIds = new TreeSet<>(ClientZookeeper.getChildren(zk,
- ClusterUtils.STORMS_SUBTREE, false));
+ ClusterUtils.STORMS_SUBTREE, false));
Set<String> activeTopologyBlobKeys = populateTopologyBlobKeys(activeTopologyIds);
Set<String> activeTopologyCodeKeys = filterTopologyCodeKeys(activeTopologyBlobKeys);
@@ -113,8 +101,8 @@ public class LeaderListenerCallback {
// this finds all active topologies blob keys from all local topology blob keys
Sets.SetView<String> diffTopology = Sets.difference(activeTopologyBlobKeys, allLocalTopologyBlobKeys);
LOG.info("active-topology-blobs [{}] local-topology-blobs [{}] diff-topology-blobs [{}]",
- generateJoinedString(activeTopologyIds), generateJoinedString(allLocalTopologyBlobKeys),
- generateJoinedString(diffTopology));
+ generateJoinedString(activeTopologyIds), generateJoinedString(allLocalTopologyBlobKeys),
+ generateJoinedString(diffTopology));
if (diffTopology.isEmpty()) {
Set<String> activeTopologyDependencies = getTopologyDependencyKeys(activeTopologyCodeKeys);
@@ -122,15 +110,15 @@ public class LeaderListenerCallback {
// this finds all dependency blob keys from active topologies from all local blob keys
Sets.SetView<String> diffDependencies = Sets.difference(activeTopologyDependencies, allLocalBlobKeys);
LOG.info("active-topology-dependencies [{}] local-blobs [{}] diff-topology-dependencies [{}]",
- generateJoinedString(activeTopologyDependencies), generateJoinedString(allLocalBlobKeys),
- generateJoinedString(diffDependencies));
+ generateJoinedString(activeTopologyDependencies), generateJoinedString(allLocalBlobKeys),
+ generateJoinedString(diffDependencies));
if (diffDependencies.isEmpty()) {
LOG.info("Accepting leadership, all active topologies and corresponding dependencies found locally.");
tc.clear();
} else {
LOG.info("Code for all active topologies is available locally, but some dependencies are not found locally, "
- + "giving up leadership.");
+ + "giving up leadership.");
closeLatch();
}
} else {
@@ -174,8 +162,8 @@ public class LeaderListenerCallback {
Set<String> topologyBlobKeys = new HashSet<>();
for (String blobKey : blobKeys) {
if (blobKey.endsWith(STORM_JAR_SUFFIX)
- || blobKey.endsWith(STORM_CODE_SUFFIX)
- || blobKey.endsWith(STORM_CONF_SUFFIX)) {
+ || blobKey.endsWith(STORM_CODE_SUFFIX)
+ || blobKey.endsWith(STORM_CONF_SUFFIX)) {
topologyBlobKeys.add(blobKey);
}
}
@@ -209,12 +197,12 @@ public class LeaderListenerCallback {
}
} catch (AuthorizationException | KeyNotFoundException | IOException e) {
LOG.error("Exception occurs while reading blob for key: "
- + activeTopologyCodeKey
- + ", exception: "
- + e, e);
+ + activeTopologyCodeKey
+ + ", exception: "
+ + e, e);
throw new RuntimeException("Exception occurs while reading blob for key: "
- + activeTopologyCodeKey
- + ", exception: " + e, e);
+ + activeTopologyCodeKey
+ + ", exception: " + e, e);
}
}
return activeTopologyDependencies;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/nimbus/NimbusHeartbeatsPressureTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/NimbusHeartbeatsPressureTest.java b/storm-server/src/main/java/org/apache/storm/nimbus/NimbusHeartbeatsPressureTest.java
index f725f5d..17d4d2f 100644
--- a/storm-server/src/main/java/org/apache/storm/nimbus/NimbusHeartbeatsPressureTest.java
+++ b/storm-server/src/main/java/org/apache/storm/nimbus/NimbusHeartbeatsPressureTest.java
@@ -1,33 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.nimbus;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
-
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import org.apache.storm.Config;
import org.apache.storm.generated.ExecutorInfo;
import org.apache.storm.generated.SupervisorWorkerHeartbeat;
@@ -51,6 +44,8 @@ public class NimbusHeartbeatsPressureTest {
private static int THREAD_SUBMIT_NUM = 1;
private static int MOCKED_STORM_NUM = 5000;
private static volatile boolean[] readyFlags = new boolean[THREADS_NUM];
+ private static Random rand = new Random(47);
+ private static List<double[]> totalCostTimesBook = new ArrayList<>();
static {
for (int i = 0; i < THREADS_NUM; i++) {
@@ -58,9 +53,6 @@ public class NimbusHeartbeatsPressureTest {
}
}
- private static Random rand = new Random(47);
- private static List<double[]> totalCostTimesBook = new ArrayList<>();
-
/**
* Initialize a fake config.
* @return conf
@@ -131,6 +123,53 @@ public class NimbusHeartbeatsPressureTest {
}
}
+ private static void printTimeCostArray(Double[] array) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("[");
+ for (int i = 0; i < array.length; i++) {
+ if (i != array.length - 1) {
+ builder.append(array[i] + ",");
+ } else {
+ builder.append(array[i] + "");
+ }
+ }
+ builder.append("]");
+ System.out.println(builder.toString());
+ }
+
+ private static void printStatistics(List<double[]> data) {
+
+ List<Double> totalPoints = new ArrayList<>();
+ double total = 0D;
+ for (double[] item : data) {
+ for (Double point : item) {
+ if (point != null) {
+ totalPoints.add(point);
+ total += point;
+ }
+ }
+ }
+ Double[] totalPointsArray = new Double[totalPoints.size()];
+
+ totalPoints.toArray(totalPointsArray);
+ Arrays.sort(totalPointsArray);
+ // printTimeCostArray(totalPointsArray);
+ println("===== statistics ================");
+ println("===== min time cost: " + totalPointsArray[0] + " =====");
+ println("===== max time cost: " + totalPointsArray[totalPointsArray.length - 2] + " =====");
+
+ double meanVal = total / totalPointsArray.length;
+ println("===== mean time cost: " + meanVal + " =====");
+ int middleIndex = (int) (totalPointsArray.length * 0.5);
+ println("===== median time cost: " + totalPointsArray[middleIndex] + " =====");
+ int top90Index = (int) (totalPointsArray.length * 0.9);
+ println("===== top90 time cost: " + totalPointsArray[top90Index] + " =====");
+ }
+
+ public static void main(String[] args) {
+ testMaxThroughput();
+ }
+
static class HeartbeatSendTask implements Runnable {
private double[] runtimesBook;
private int taskId;
@@ -191,51 +230,4 @@ public class NimbusHeartbeatsPressureTest {
}
}
- private static void printTimeCostArray(Double[] array) {
- StringBuilder builder = new StringBuilder();
- builder.append("[");
- for (int i = 0; i < array.length; i++) {
- if (i != array.length - 1) {
- builder.append(array[i] + ",");
- } else {
- builder.append(array[i] + "");
- }
- }
- builder.append("]");
- System.out.println(builder.toString());
- }
-
- private static void printStatistics(List<double[]> data) {
-
- List<Double> totalPoints = new ArrayList<>();
- double total = 0D;
- for (double[] item : data) {
- for (Double point : item) {
- if (point != null) {
- totalPoints.add(point);
- total += point;
- }
- }
- }
- Double[] totalPointsArray = new Double[totalPoints.size()];
-
- totalPoints.toArray(totalPointsArray);
- Arrays.sort(totalPointsArray);
- // printTimeCostArray(totalPointsArray);
- println("===== statistics ================");
- println("===== min time cost: " + totalPointsArray[0] + " =====");
- println("===== max time cost: " + totalPointsArray[totalPointsArray.length - 2] + " =====");
-
- double meanVal = total / totalPointsArray.length;
- println("===== mean time cost: " + meanVal + " =====");
- int middleIndex = (int) (totalPointsArray.length * 0.5);
- println("===== median time cost: " + totalPointsArray[middleIndex] + " =====");
- int top90Index = (int) (totalPointsArray.length * 0.9);
- println("===== top90 time cost: " + totalPointsArray[top90Index] + " =====");
- }
-
- public static void main(String[] args) {
- testMaxThroughput();
- }
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/nimbus/StrictTopologyValidator.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/StrictTopologyValidator.java b/storm-server/src/main/java/org/apache/storm/nimbus/StrictTopologyValidator.java
index cdf22c7..b7e7117 100644
--- a/storm-server/src/main/java/org/apache/storm/nimbus/StrictTopologyValidator.java
+++ b/storm-server/src/main/java/org/apache/storm/nimbus/StrictTopologyValidator.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.nimbus;
@@ -30,7 +24,7 @@ public class StrictTopologyValidator implements ITopologyValidator {
private static final Logger LOG = LoggerFactory.getLogger(StrictTopologyValidator.class);
@Override
- public void prepare(Map stormConf){
+ public void prepare(Map stormConf) {
}
@Override
@@ -63,5 +57,5 @@ public class StrictTopologyValidator implements ITopologyValidator {
}
}
}
- }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/nimbus/TimeOutWorkerHeartbeatsRecoveryStrategy.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/TimeOutWorkerHeartbeatsRecoveryStrategy.java b/storm-server/src/main/java/org/apache/storm/nimbus/TimeOutWorkerHeartbeatsRecoveryStrategy.java
index 78f880a..ac01aae 100644
--- a/storm-server/src/main/java/org/apache/storm/nimbus/TimeOutWorkerHeartbeatsRecoveryStrategy.java
+++ b/storm-server/src/main/java/org/apache/storm/nimbus/TimeOutWorkerHeartbeatsRecoveryStrategy.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.nimbus;
@@ -21,11 +15,9 @@ package org.apache.storm.nimbus;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-
import org.apache.storm.Config;
import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.Time;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/nimbus/WorkerHeartbeatsRecoveryStrategyFactory.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/WorkerHeartbeatsRecoveryStrategyFactory.java b/storm-server/src/main/java/org/apache/storm/nimbus/WorkerHeartbeatsRecoveryStrategyFactory.java
index 4befaf8..f656089 100644
--- a/storm-server/src/main/java/org/apache/storm/nimbus/WorkerHeartbeatsRecoveryStrategyFactory.java
+++ b/storm-server/src/main/java/org/apache/storm/nimbus/WorkerHeartbeatsRecoveryStrategyFactory.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.nimbus;
@@ -37,10 +31,10 @@ public class WorkerHeartbeatsRecoveryStrategyFactory {
IWorkerHeartbeatsRecoveryStrategy strategy;
if (conf.get(DaemonConfig.NIMBUS_WORKER_HEARTBEATS_RECOVERY_STRATEGY_CLASS) != null) {
Object targetObj = ReflectionUtils.newInstance((String)
- conf.get(DaemonConfig.NIMBUS_WORKER_HEARTBEATS_RECOVERY_STRATEGY_CLASS));
+ conf.get(DaemonConfig.NIMBUS_WORKER_HEARTBEATS_RECOVERY_STRATEGY_CLASS));
Preconditions.checkState(targetObj instanceof IWorkerHeartbeatsRecoveryStrategy,
- "{} must implements IWorkerHeartbeatsRecoveryStrategy",
- DaemonConfig.NIMBUS_WORKER_HEARTBEATS_RECOVERY_STRATEGY_CLASS);
+ "{} must implements IWorkerHeartbeatsRecoveryStrategy",
+ DaemonConfig.NIMBUS_WORKER_HEARTBEATS_RECOVERY_STRATEGY_CLASS);
strategy = ((IWorkerHeartbeatsRecoveryStrategy) targetObj);
} else {
strategy = new TimeOutWorkerHeartbeatsRecoveryStrategy();
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/pacemaker/IServerMessageHandler.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/pacemaker/IServerMessageHandler.java b/storm-server/src/main/java/org/apache/storm/pacemaker/IServerMessageHandler.java
index 7dca237..a7ebc1e 100644
--- a/storm-server/src/main/java/org/apache/storm/pacemaker/IServerMessageHandler.java
+++ b/storm-server/src/main/java/org/apache/storm/pacemaker/IServerMessageHandler.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.pacemaker;
import org.apache.storm.generated.HBMessage;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/pacemaker/Pacemaker.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/pacemaker/Pacemaker.java b/storm-server/src/main/java/org/apache/storm/pacemaker/Pacemaker.java
index 095982b..ff283a4 100644
--- a/storm-server/src/main/java/org/apache/storm/pacemaker/Pacemaker.java
+++ b/storm-server/src/main/java/org/apache/storm/pacemaker/Pacemaker.java
@@ -1,29 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.pacemaker;
+import com.codahale.metrics.ExponentiallyDecayingReservoir;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
-
import org.apache.storm.generated.HBMessage;
import org.apache.storm.generated.HBMessageData;
import org.apache.storm.generated.HBNodes;
@@ -35,79 +32,81 @@ import org.apache.storm.utils.Utils;
import org.apache.storm.utils.VersionInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-import com.codahale.metrics.ExponentiallyDecayingReservoir;
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.Meter;
-
import uk.org.lidalia.sysoutslf4j.context.SysOutOverSLF4J;
public class Pacemaker implements IServerMessageHandler {
private static final Logger LOG = LoggerFactory.getLogger(Pacemaker.class);
- private Map<String, byte[]> heartbeats;
- private Map<String, Object> conf;
-
private final static Meter meterSendPulseCount = StormMetricsRegistry.registerMeter("pacemaker:send-pulse-count");
private final static Meter meterTotalReceivedSize = StormMetricsRegistry.registerMeter("pacemaker:total-receive-size");
private final static Meter meterGetPulseCount = StormMetricsRegistry.registerMeter("pacemaker:get-pulse=count");
private final static Meter meterTotalSentSize = StormMetricsRegistry.registerMeter("pacemaker:total-sent-size");
- private final static Histogram histogramHeartbeatSize = StormMetricsRegistry.registerHistogram("pacemaker:heartbeat-size", new ExponentiallyDecayingReservoir());
+ private final static Histogram histogramHeartbeatSize =
+ StormMetricsRegistry.registerHistogram("pacemaker:heartbeat-size", new ExponentiallyDecayingReservoir());
+ private Map<String, byte[]> heartbeats;
+ private Map<String, Object> conf;
public Pacemaker(Map<String, Object> conf) {
heartbeats = new ConcurrentHashMap();
this.conf = conf;
StormMetricsRegistry.registerGauge("pacemaker:size-total-keys",
- new Callable() {
- @Override
- public Integer call() throws Exception {
- return heartbeats.size();
- }
- });
+ new Callable() {
+ @Override
+ public Integer call() throws Exception {
+ return heartbeats.size();
+ }
+ });
StormMetricsRegistry.startMetricsReporters(conf);
}
+ public static void main(String[] args) {
+ SysOutOverSLF4J.sendSystemOutAndErrToSLF4J();
+ Map<String, Object> conf = ConfigUtils.overrideLoginConfigWithSystemProperty(Utils.readStormConfig());
+ final Pacemaker serverHandler = new Pacemaker(conf);
+ serverHandler.launchServer();
+ }
+
@Override
public HBMessage handleMessage(HBMessage m, boolean authenticated) {
HBMessage response = null;
HBMessageData data = m.get_data();
switch (m.get_type()) {
- case CREATE_PATH:
- response = createPath(data.get_path());
- break;
- case EXISTS:
- response = pathExists(data.get_path(), authenticated);
- break;
- case SEND_PULSE:
- response = sendPulse(data.get_pulse());
- break;
- case GET_ALL_PULSE_FOR_PATH:
- response = getAllPulseForPath(data.get_path(), authenticated);
- break;
- case GET_ALL_NODES_FOR_PATH:
- response = getAllNodesForPath(data.get_path(), authenticated);
- break;
- case GET_PULSE:
- response = getPulse(data.get_path(), authenticated);
- break;
- case DELETE_PATH:
- response = deletePath(data.get_path());
- break;
- case DELETE_PULSE_ID:
- response = deletePulseId(data.get_path());
- break;
- default:
- LOG.info("Got Unexpected Type: {}", m.get_type());
- break;
+ case CREATE_PATH:
+ response = createPath(data.get_path());
+ break;
+ case EXISTS:
+ response = pathExists(data.get_path(), authenticated);
+ break;
+ case SEND_PULSE:
+ response = sendPulse(data.get_pulse());
+ break;
+ case GET_ALL_PULSE_FOR_PATH:
+ response = getAllPulseForPath(data.get_path(), authenticated);
+ break;
+ case GET_ALL_NODES_FOR_PATH:
+ response = getAllNodesForPath(data.get_path(), authenticated);
+ break;
+ case GET_PULSE:
+ response = getPulse(data.get_path(), authenticated);
+ break;
+ case DELETE_PATH:
+ response = deletePath(data.get_path());
+ break;
+ case DELETE_PULSE_ID:
+ response = deletePulseId(data.get_path());
+ break;
+ default:
+ LOG.info("Got Unexpected Type: {}", m.get_type());
+ break;
}
- if (response != null)
+ if (response != null) {
response.set_message_id(m.get_message_id());
+ }
return response;
}
-
private HBMessage createPath(String path) {
return new HBMessage(HBServerMessageType.CREATE_PATH_RESPONSE, null);
}
@@ -192,8 +191,9 @@ public class Pacemaker implements IServerMessageHandler {
String prefix = path.endsWith("/") ? path : (path + "/");
for (String key : heartbeats.keySet()) {
String checkKey = key + "/";
- if (checkKey.indexOf(prefix) == 0)
+ if (checkKey.indexOf(prefix) == 0) {
deletePulseId(key);
+ }
}
return new HBMessage(HBServerMessageType.DELETE_PATH_RESPONSE, null);
}
@@ -209,11 +209,4 @@ public class Pacemaker implements IServerMessageHandler {
return new PacemakerServer(this, conf);
}
- public static void main(String[] args) {
- SysOutOverSLF4J.sendSystemOutAndErrToSLF4J();
- Map<String, Object> conf = ConfigUtils.overrideLoginConfigWithSystemProperty(Utils.readStormConfig());
- final Pacemaker serverHandler = new Pacemaker(conf);
- serverHandler.launchServer();
- }
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/pacemaker/PacemakerServer.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/pacemaker/PacemakerServer.java b/storm-server/src/main/java/org/apache/storm/pacemaker/PacemakerServer.java
index 2f050db..e4e3c67 100644
--- a/storm-server/src/main/java/org/apache/storm/pacemaker/PacemakerServer.java
+++ b/storm-server/src/main/java/org/apache/storm/pacemaker/PacemakerServer.java
@@ -1,36 +1,30 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.pacemaker;
-import org.apache.storm.Config;
-import org.apache.storm.DaemonConfig;
-import org.apache.storm.generated.HBMessage;
-import org.apache.storm.messaging.netty.ISaslServer;
-import org.apache.storm.messaging.netty.NettyRenameThreadFactory;
-import org.apache.storm.security.auth.AuthUtils;
-import java.lang.InterruptedException;
import java.net.InetSocketAddress;
import java.util.Map;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import javax.security.auth.login.Configuration;
+import org.apache.storm.Config;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.generated.HBMessage;
+import org.apache.storm.messaging.netty.ISaslServer;
+import org.apache.storm.messaging.netty.NettyRenameThreadFactory;
import org.apache.storm.pacemaker.codec.ThriftNettyServerCodec;
+import org.apache.storm.security.auth.AuthUtils;
import org.jboss.netty.bootstrap.ServerBootstrap;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelPipelineFactory;
@@ -55,48 +49,47 @@ class PacemakerServer implements ISaslServer {
private ConcurrentSkipListSet<Channel> authenticated_channels = new ConcurrentSkipListSet<Channel>();
private ThriftNettyServerCodec.AuthMethod authMethod;
- public PacemakerServer(IServerMessageHandler handler, Map<String, Object> config){
- int maxWorkers = (int)config.get(DaemonConfig.PACEMAKER_MAX_THREADS);
- this.port = (int)config.get(Config.PACEMAKER_PORT);
+ public PacemakerServer(IServerMessageHandler handler, Map<String, Object> config) {
+ int maxWorkers = (int) config.get(DaemonConfig.PACEMAKER_MAX_THREADS);
+ this.port = (int) config.get(Config.PACEMAKER_PORT);
this.handler = handler;
this.topo_name = "pacemaker_server";
- String auth = (String)config.get(Config.PACEMAKER_AUTH_METHOD);
- switch(auth) {
-
- case "DIGEST":
- Configuration login_conf = AuthUtils.GetConfiguration(config);
- authMethod = ThriftNettyServerCodec.AuthMethod.DIGEST;
- this.secret = AuthUtils.makeDigestPayload(login_conf, AuthUtils.LOGIN_CONTEXT_PACEMAKER_DIGEST);
- if(this.secret == null) {
- LOG.error("Can't start pacemaker server without digest secret.");
- throw new RuntimeException("Can't start pacemaker server without digest secret.");
- }
- break;
-
- case "KERBEROS":
- authMethod = ThriftNettyServerCodec.AuthMethod.KERBEROS;
- break;
-
- case "NONE":
- authMethod = ThriftNettyServerCodec.AuthMethod.NONE;
- break;
-
- default:
- LOG.error("Can't start pacemaker server without proper PACEMAKER_AUTH_METHOD.");
- throw new RuntimeException("Can't start pacemaker server without proper PACEMAKER_AUTH_METHOD.");
+ String auth = (String) config.get(Config.PACEMAKER_AUTH_METHOD);
+ switch (auth) {
+
+ case "DIGEST":
+ Configuration login_conf = AuthUtils.GetConfiguration(config);
+ authMethod = ThriftNettyServerCodec.AuthMethod.DIGEST;
+ this.secret = AuthUtils.makeDigestPayload(login_conf, AuthUtils.LOGIN_CONTEXT_PACEMAKER_DIGEST);
+ if (this.secret == null) {
+ LOG.error("Can't start pacemaker server without digest secret.");
+ throw new RuntimeException("Can't start pacemaker server without digest secret.");
+ }
+ break;
+
+ case "KERBEROS":
+ authMethod = ThriftNettyServerCodec.AuthMethod.KERBEROS;
+ break;
+
+ case "NONE":
+ authMethod = ThriftNettyServerCodec.AuthMethod.NONE;
+ break;
+
+ default:
+ LOG.error("Can't start pacemaker server without proper PACEMAKER_AUTH_METHOD.");
+ throw new RuntimeException("Can't start pacemaker server without proper PACEMAKER_AUTH_METHOD.");
}
ThreadFactory bossFactory = new NettyRenameThreadFactory("server-boss");
ThreadFactory workerFactory = new NettyRenameThreadFactory("server-worker");
NioServerSocketChannelFactory factory;
- if(maxWorkers > 0) {
+ if (maxWorkers > 0) {
factory =
new NioServerSocketChannelFactory(Executors.newCachedThreadPool(bossFactory),
Executors.newCachedThreadPool(workerFactory),
maxWorkers);
- }
- else {
+ } else {
factory =
new NioServerSocketChannelFactory(Executors.newCachedThreadPool(bossFactory),
Executors.newCachedThreadPool(workerFactory));
@@ -106,10 +99,10 @@ class PacemakerServer implements ISaslServer {
bootstrap.setOption("tcpNoDelay", true);
bootstrap.setOption("sendBufferSize", FIVE_MB_IN_BYTES);
bootstrap.setOption("keepAlive", true);
- int thriftMessageMaxSize = (Integer)config.get(Config.PACEMAKER_THRIFT_MESSAGE_SIZE_MAX);
+ int thriftMessageMaxSize = (Integer) config.get(Config.PACEMAKER_THRIFT_MESSAGE_SIZE_MAX);
ChannelPipelineFactory pipelineFactory =
new ThriftNettyServerCodec(this, config, authMethod, thriftMessageMaxSize)
- .pipelineFactory();
+ .pipelineFactory();
bootstrap.setPipelineFactory(pipelineFactory);
Channel channel = bootstrap.bind(new InetSocketAddress(port));
allChannels.add(channel);
@@ -123,11 +116,10 @@ class PacemakerServer implements ISaslServer {
public void cleanPipeline(Channel channel) {
boolean authenticated = authenticated_channels.contains(channel);
- if(!authenticated) {
- if(channel.getPipeline().get(ThriftNettyServerCodec.SASL_HANDLER) != null) {
+ if (!authenticated) {
+ if (channel.getPipeline().get(ThriftNettyServerCodec.SASL_HANDLER) != null) {
channel.getPipeline().remove(ThriftNettyServerCodec.SASL_HANDLER);
- }
- else if(channel.getPipeline().get(ThriftNettyServerCodec.KERBEROS_HANDLER) != null) {
+ } else if (channel.getPipeline().get(ThriftNettyServerCodec.KERBEROS_HANDLER) != null) {
channel.getPipeline().remove(ThriftNettyServerCodec.KERBEROS_HANDLER);
}
}
@@ -137,15 +129,14 @@ class PacemakerServer implements ISaslServer {
cleanPipeline(channel);
boolean authenticated = (authMethod == ThriftNettyServerCodec.AuthMethod.NONE) || authenticated_channels.contains(channel);
- HBMessage m = (HBMessage)mesg;
+ HBMessage m = (HBMessage) mesg;
LOG.debug("received message. Passing to handler. {} : {} : {}",
handler.toString(), m.toString(), channel.toString());
HBMessage response = handler.handleMessage(m, authenticated);
- if(response != null) {
+ if (response != null) {
LOG.debug("Got Response from handler: {}", response);
channel.write(response);
- }
- else {
+ } else {
LOG.info("Got null response from handler handling message: {}", m);
}
}
[41/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-starter
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java
index 4da1c85..1e57a94 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ReachTopology.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import java.util.Arrays;
@@ -23,7 +18,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.coordination.BatchOutputCollector;
@@ -71,6 +65,34 @@ public class ReachTopology {
put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob"));
}};
+ public static LinearDRPCTopologyBuilder construct() {
+ LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("reach");
+ builder.addBolt(new GetTweeters(), 4);
+ builder.addBolt(new GetFollowers(), 12).shuffleGrouping();
+ builder.addBolt(new PartialUniquer(), 6).fieldsGrouping(new Fields("id", "follower"));
+ builder.addBolt(new CountAggregator(), 3).fieldsGrouping(new Fields("id"));
+ return builder;
+ }
+
+ public static void main(String[] args) throws Exception {
+ LinearDRPCTopologyBuilder builder = construct();
+
+ Config conf = new Config();
+ conf.setNumWorkers(6);
+ String topoName = "reach-drpc";
+ if (args.length > 0) {
+ topoName = args[0];
+ }
+ StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createRemoteTopology());
+
+ try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
+ String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
+ for (String url : urlsToTry) {
+ System.out.println("Reach of " + url + ": " + drpc.execute("reach", url));
+ }
+ }
+ }
+
public static class GetTweeters extends BaseBasicBolt {
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
@@ -162,32 +184,4 @@ public class ReachTopology {
declarer.declare(new Fields("id", "reach"));
}
}
-
- public static LinearDRPCTopologyBuilder construct() {
- LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("reach");
- builder.addBolt(new GetTweeters(), 4);
- builder.addBolt(new GetFollowers(), 12).shuffleGrouping();
- builder.addBolt(new PartialUniquer(), 6).fieldsGrouping(new Fields("id", "follower"));
- builder.addBolt(new CountAggregator(), 3).fieldsGrouping(new Fields("id"));
- return builder;
- }
-
- public static void main(String[] args) throws Exception {
- LinearDRPCTopologyBuilder builder = construct();
-
- Config conf = new Config();
- conf.setNumWorkers(6);
- String topoName = "reach-drpc";
- if (args.length > 0) {
- topoName = args[0];
- }
- StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createRemoteTopology());
-
- try (DRPCClient drpc = DRPCClient.getConfiguredClient(conf)) {
- String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
- for (String url : urlsToTry) {
- System.out.println("Reach of " + url + ": " + drpc.execute("reach", url));
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java
index 07149e8..18af7ea 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/ResourceAwareExampleTopology.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.starter;
@@ -22,7 +16,6 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.task.OutputCollector;
@@ -39,53 +32,6 @@ import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
public class ResourceAwareExampleTopology {
- public static class ExclamationBolt extends BaseRichBolt {
- //Have a crummy cache to show off shared memory accounting
- private static final ConcurrentHashMap<String, String> myCrummyCache =
- new ConcurrentHashMap<>();
- private static final int CACHE_SIZE = 100_000;
- OutputCollector _collector;
-
- protected static String getFromCache(String key) {
- return myCrummyCache.get(key);
- }
-
- protected static void addToCache(String key, String value) {
- myCrummyCache.putIfAbsent(key, value);
- int numToRemove = myCrummyCache.size() - CACHE_SIZE;
- if (numToRemove > 0) {
- //Remove something randomly...
- Iterator<Entry<String, String>> it = myCrummyCache.entrySet().iterator();
- for (; numToRemove > 0 && it.hasNext(); numToRemove--) {
- it.next();
- it.remove();
- }
- }
- }
-
- @Override
- public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
- _collector = collector;
- }
-
- @Override
- public void execute(Tuple tuple) {
- String orig = tuple.getString(0);
- String ret = getFromCache(orig);
- if (ret == null) {
- ret = orig + "!!!";
- addToCache(orig, ret);
- }
- _collector.emit(tuple, new Values(ret));
- _collector.ack(tuple);
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("word"));
- }
- }
-
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
@@ -154,4 +100,51 @@ public class ResourceAwareExampleTopology {
StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
+
+ public static class ExclamationBolt extends BaseRichBolt {
+ //Have a crummy cache to show off shared memory accounting
+ private static final ConcurrentHashMap<String, String> myCrummyCache =
+ new ConcurrentHashMap<>();
+ private static final int CACHE_SIZE = 100_000;
+ OutputCollector _collector;
+
+ protected static String getFromCache(String key) {
+ return myCrummyCache.get(key);
+ }
+
+ protected static void addToCache(String key, String value) {
+ myCrummyCache.putIfAbsent(key, value);
+ int numToRemove = myCrummyCache.size() - CACHE_SIZE;
+ if (numToRemove > 0) {
+ //Remove something randomly...
+ Iterator<Entry<String, String>> it = myCrummyCache.entrySet().iterator();
+ for (; numToRemove > 0 && it.hasNext(); numToRemove--) {
+ it.next();
+ it.remove();
+ }
+ }
+ }
+
+ @Override
+ public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
+ _collector = collector;
+ }
+
+ @Override
+ public void execute(Tuple tuple) {
+ String orig = tuple.getString(0);
+ String ret = getFromCache(orig);
+ if (ret == null) {
+ ret = orig + "!!!";
+ addToCache(orig, ret);
+ }
+ _collector.emit(tuple, new Values(ret));
+ _collector.ack(tuple);
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word"));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java
index 78b2baf..23f56c2 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/RollingTopWords.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import org.apache.storm.starter.bolt.IntermediateRankingsBolt;
@@ -35,54 +30,54 @@ import org.slf4j.LoggerFactory;
*/
public class RollingTopWords extends ConfigurableTopology {
- private static final Logger LOG = LoggerFactory.getLogger(RollingTopWords.class);
- private static final int TOP_N = 5;
-
- private RollingTopWords() {
- }
+ private static final Logger LOG = LoggerFactory.getLogger(RollingTopWords.class);
+ private static final int TOP_N = 5;
- public static void main(String[] args) throws Exception {
- ConfigurableTopology.start(new RollingTopWords(), args);
- }
+ private RollingTopWords() {
+ }
- /**
- * Submits (runs) the topology.
- *
- * Usage: "RollingTopWords [topology-name] [-local]"
- *
- * By default, the topology is run locally under the name
- * "slidingWindowCounts".
- *
- * Examples:
- *
- * ```
- *
- * # Runs in remote/cluster mode, with topology name "production-topology"
- * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords production-topology ```
- *
- * @param args
- * First positional argument (optional) is topology name, second
- * positional argument (optional) defines whether to run the topology
- * locally ("-local") or remotely, i.e. on a real cluster.
- * @throws Exception
- */
- protected int run(String[] args) {
- String topologyName = "slidingWindowCounts";
- if (args.length >= 1) {
- topologyName = args[0];
+ public static void main(String[] args) throws Exception {
+ ConfigurableTopology.start(new RollingTopWords(), args);
}
- TopologyBuilder builder = new TopologyBuilder();
- String spoutId = "wordGenerator";
- String counterId = "counter";
- String intermediateRankerId = "intermediateRanker";
- String totalRankerId = "finalRanker";
- builder.setSpout(spoutId, new TestWordSpout(), 5);
- builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word"));
- builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId,
- new Fields("obj"));
- builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
- LOG.info("Topology name: " + topologyName);
- return submit(topologyName, conf, builder);
- }
+ /**
+ * Submits (runs) the topology.
+ *
+ * Usage: "RollingTopWords [topology-name] [-local]"
+ *
+ * By default, the topology is run locally under the name
+ * "slidingWindowCounts".
+ *
+ * Examples:
+ *
+ * ```
+ *
+ * # Runs in remote/cluster mode, with topology name "production-topology"
+ * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords production-topology ```
+ *
+ * @param args
+ * First positional argument (optional) is topology name, second
+ * positional argument (optional) defines whether to run the topology
+ * locally ("-local") or remotely, i.e. on a real cluster.
+ * @throws Exception
+ */
+ protected int run(String[] args) {
+ String topologyName = "slidingWindowCounts";
+ if (args.length >= 1) {
+ topologyName = args[0];
+ }
+ TopologyBuilder builder = new TopologyBuilder();
+ String spoutId = "wordGenerator";
+ String counterId = "counter";
+ String intermediateRankerId = "intermediateRanker";
+ String totalRankerId = "finalRanker";
+ builder.setSpout(spoutId, new TestWordSpout(), 5);
+ builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word"));
+ builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId,
+ new Fields("obj"));
+ builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
+ LOG.info("Topology name: " + topologyName);
+
+ return submit(topologyName, conf, builder);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java
index 2e1bb94..4f4b6b6 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SingleJoinExample.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import org.apache.storm.Config;
@@ -34,7 +29,7 @@ public class SingleJoinExample {
public static void main(String[] args) throws Exception {
if (!NimbusClient.isLocalOverride()) {
throw new IllegalStateException("This example only works in local mode. "
- + "Run with storm local not storm jar");
+ + "Run with storm local not storm jar");
}
FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));
@@ -43,7 +38,7 @@ public class SingleJoinExample {
builder.setSpout("gender", genderSpout);
builder.setSpout("age", ageSpout);
builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))).fieldsGrouping("gender", new Fields("id"))
- .fieldsGrouping("age", new Fields("id"));
+ .fieldsGrouping("age", new Fields("id"));
Config conf = new Config();
conf.setDebug(true);
@@ -53,8 +48,7 @@ public class SingleJoinExample {
String gender;
if (i % 2 == 0) {
gender = "male";
- }
- else {
+ } else {
gender = "female";
}
genderSpout.feed(new Values(i, gender));
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java
index 4e18217..a3b8296 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SkewedRollingTopWords.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import org.apache.storm.starter.bolt.IntermediateRankingsBolt;
@@ -38,55 +33,55 @@ import org.slf4j.LoggerFactory;
*/
public class SkewedRollingTopWords extends ConfigurableTopology {
- private static final Logger LOG = LoggerFactory.getLogger(SkewedRollingTopWords.class);
- private static final int TOP_N = 5;
-
- private SkewedRollingTopWords() {
- }
+ private static final Logger LOG = LoggerFactory.getLogger(SkewedRollingTopWords.class);
+ private static final int TOP_N = 5;
- public static void main(String[] args) throws Exception {
- ConfigurableTopology.start(new SkewedRollingTopWords(), args);
- }
+ private SkewedRollingTopWords() {
+ }
- /**
- * Submits (runs) the topology.
- *
- * Usage: "SkewedRollingTopWords [topology-name] [-local]"
- *
- * By default, the topology is run locally under the name
- * "slidingWindowCounts".
- *
- * Examples:
- *
- * ```
- *
- * # Runs in remote/cluster mode, with topology name "production-topology"
- * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.SkewedRollingTopWords production-topology ```
- *
- * @param args
- * First positional argument (optional) is topology name, second
- * positional argument (optional) defines whether to run the topology
- * locally ("-local") or remotely, i.e. on a real cluster.
- * @throws Exception
- */
- protected int run(String[] args) {
- String topologyName = "slidingWindowCounts";
- if (args.length >= 1) {
- topologyName = args[0];
+ public static void main(String[] args) throws Exception {
+ ConfigurableTopology.start(new SkewedRollingTopWords(), args);
}
- TopologyBuilder builder = new TopologyBuilder();
- String spoutId = "wordGenerator";
- String counterId = "counter";
- String aggId = "aggregator";
- String intermediateRankerId = "intermediateRanker";
- String totalRankerId = "finalRanker";
- builder.setSpout(spoutId, new TestWordSpout(), 5);
- builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word"));
- builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj"));
- builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj"));
- builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
- LOG.info("Topology name: " + topologyName);
- return submit(topologyName, conf, builder);
- }
+ /**
+ * Submits (runs) the topology.
+ *
+ * Usage: "SkewedRollingTopWords [topology-name] [-local]"
+ *
+ * By default, the topology is run locally under the name
+ * "slidingWindowCounts".
+ *
+ * Examples:
+ *
+ * ```
+ *
+ * # Runs in remote/cluster mode, with topology name "production-topology"
+ * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.SkewedRollingTopWords production-topology ```
+ *
+ * @param args
+ * First positional argument (optional) is topology name, second
+ * positional argument (optional) defines whether to run the topology
+ * locally ("-local") or remotely, i.e. on a real cluster.
+ * @throws Exception
+ */
+ protected int run(String[] args) {
+ String topologyName = "slidingWindowCounts";
+ if (args.length >= 1) {
+ topologyName = args[0];
+ }
+ TopologyBuilder builder = new TopologyBuilder();
+ String spoutId = "wordGenerator";
+ String counterId = "counter";
+ String aggId = "aggregator";
+ String intermediateRankerId = "intermediateRanker";
+ String totalRankerId = "finalRanker";
+ builder.setSpout(spoutId, new TestWordSpout(), 5);
+ builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word"));
+ builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj"));
+ builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj"));
+ builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
+ LOG.info("Topology name: " + topologyName);
+
+ return submit(topologyName, conf, builder);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java
index 6204f8c..d179899 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingTupleTsTopology.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import java.util.concurrent.TimeUnit;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.starter.bolt.PrinterBolt;
@@ -36,20 +30,20 @@ public class SlidingTupleTsTopology {
public static void main(String[] args) throws Exception {
TopologyBuilder builder = new TopologyBuilder();
BaseWindowedBolt bolt = new SlidingWindowSumBolt()
- .withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS))
- .withTimestampField("ts")
- .withLag(new Duration(5, TimeUnit.SECONDS));
+ .withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS))
+ .withTimestampField("ts")
+ .withLag(new Duration(5, TimeUnit.SECONDS));
builder.setSpout("integer", new RandomIntegerSpout(), 1);
builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer");
builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum");
Config conf = new Config();
conf.setDebug(true);
String topoName = "test";
-
+
if (args != null && args.length > 0) {
topoName = args[0];
}
-
+
conf.setNumWorkers(1);
StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java
index fc7cf4a..a75f999 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/SlidingWindowTopology.java
@@ -1,26 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.starter;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.starter.bolt.PrinterBolt;
@@ -47,6 +40,24 @@ public class SlidingWindowTopology {
private static final Logger LOG = LoggerFactory.getLogger(SlidingWindowTopology.class);
+ public static void main(String[] args) throws Exception {
+ TopologyBuilder builder = new TopologyBuilder();
+ builder.setSpout("integer", new RandomIntegerSpout(), 1);
+ builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(Count.of(30), Count.of(10)), 1)
+ .shuffleGrouping("integer");
+ builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(Count.of(3)), 1)
+ .shuffleGrouping("slidingsum");
+ builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg");
+ Config conf = new Config();
+ conf.setDebug(true);
+ String topoName = "test";
+ if (args != null && args.length > 0) {
+ topoName = args[0];
+ }
+ conf.setNumWorkers(1);
+ StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
+ }
+
/*
* Computes tumbling window average
*/
@@ -65,9 +76,9 @@ public class SlidingWindowTopology {
LOG.debug("Events in current window: " + tuplesInWindow.size());
if (tuplesInWindow.size() > 0) {
/*
- * Since this is a tumbling window calculation,
- * we use all the tuples in the window to compute the avg.
- */
+ * Since this is a tumbling window calculation,
+ * we use all the tuples in the window to compute the avg.
+ */
for (Tuple tuple : tuplesInWindow) {
sum += (int) tuple.getValue(0);
}
@@ -80,23 +91,4 @@ public class SlidingWindowTopology {
declarer.declare(new Fields("avg"));
}
}
-
-
- public static void main(String[] args) throws Exception {
- TopologyBuilder builder = new TopologyBuilder();
- builder.setSpout("integer", new RandomIntegerSpout(), 1);
- builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(Count.of(30), Count.of(10)), 1)
- .shuffleGrouping("integer");
- builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(Count.of(3)), 1)
- .shuffleGrouping("slidingsum");
- builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg");
- Config conf = new Config();
- conf.setDebug(true);
- String topoName = "test";
- if (args != null && args.length > 0) {
- topoName = args[0];
- }
- conf.setNumWorkers(1);
- StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java
index e407ce8..3f30f48 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulTopology.java
@@ -19,7 +19,6 @@
package org.apache.storm.starter;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.starter.spout.RandomIntegerSpout;
@@ -38,20 +37,19 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * An example topology that demonstrates the use of {@link org.apache.storm.topology.IStatefulBolt}
- * to manage state. To run the example,
+ * An example topology that demonstrates the use of {@link org.apache.storm.topology.IStatefulBolt} to manage state. To run the example,
* <pre>
* $ storm jar examples/storm-starter/storm-starter-topologies-*.jar storm.starter.StatefulTopology statetopology
* </pre>
* <p/>
- * The default state used is 'InMemoryKeyValueState' which does not persist the state across restarts. You could use
- * 'RedisKeyValueState' to test state persistence by setting below property in conf/storm.yaml
+ * The default state used is 'InMemoryKeyValueState' which does not persist the state across restarts. You could use 'RedisKeyValueState' to
+ * test state persistence by setting below property in conf/storm.yaml
* <pre>
* topology.state.provider: org.apache.storm.redis.state.RedisKeyValueStateProvider
* </pre>
* <p/>
- * You should also start a local redis instance before running the 'storm jar' command. The default
- * RedisKeyValueStateProvider parameters can be overridden in conf/storm.yaml, for e.g.
+ * You should also start a local redis instance before running the 'storm jar' command. The default RedisKeyValueStateProvider parameters
+ * can be overridden in conf/storm.yaml, for e.g.
* <p/>
* <pre>
* topology.state.provider.config: '{"keyClass":"...", "valueClass":"...",
@@ -65,6 +63,22 @@ import org.slf4j.LoggerFactory;
public class StatefulTopology {
private static final Logger LOG = LoggerFactory.getLogger(StatefulTopology.class);
+ public static void main(String[] args) throws Exception {
+ TopologyBuilder builder = new TopologyBuilder();
+ builder.setSpout("spout", new RandomIntegerSpout());
+ builder.setBolt("partialsum", new StatefulSumBolt("partial"), 1).shuffleGrouping("spout");
+ builder.setBolt("printer", new PrinterBolt(), 2).shuffleGrouping("partialsum");
+ builder.setBolt("total", new StatefulSumBolt("total"), 1).shuffleGrouping("printer");
+ Config conf = new Config();
+ conf.setDebug(false);
+ String topoName = "test";
+ if (args != null && args.length > 0) {
+ topoName = args[0];
+ }
+ conf.setNumWorkers(1);
+ StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
+ }
+
/**
* A bolt that uses {@link KeyValueState} to save its state.
*/
@@ -119,20 +133,4 @@ public class StatefulTopology {
}
}
-
- public static void main(String[] args) throws Exception {
- TopologyBuilder builder = new TopologyBuilder();
- builder.setSpout("spout", new RandomIntegerSpout());
- builder.setBolt("partialsum", new StatefulSumBolt("partial"), 1).shuffleGrouping("spout");
- builder.setBolt("printer", new PrinterBolt(), 2).shuffleGrouping("partialsum");
- builder.setBolt("total", new StatefulSumBolt("total"), 1).shuffleGrouping("printer");
- Config conf = new Config();
- conf.setDebug(false);
- String topoName = "test";
- if (args != null && args.length > 0) {
- topoName = args[0];
- }
- conf.setNumWorkers(1);
- StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java
index 30c1ab2..eb0132e 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/StatefulWindowingTopology.java
@@ -19,7 +19,6 @@
package org.apache.storm.starter;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.starter.bolt.PrinterBolt;
@@ -40,18 +39,37 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * A simple example that demonstrates the usage of {@link org.apache.storm.topology.IStatefulWindowedBolt} to
- * save the state of the windowing operation to avoid re-computation in case of failures.
+ * A simple example that demonstrates the usage of {@link org.apache.storm.topology.IStatefulWindowedBolt} to save the state of the
+ * windowing operation to avoid re-computation in case of failures.
* <p>
* The framework internally manages the window boundaries and does not invoke
- * {@link org.apache.storm.topology.IWindowedBolt#execute(TupleWindow)} for the already evaluated windows in case of restarts
- * during failures. The {@link org.apache.storm.topology.IStatefulBolt#initState(State)}
+ * {@link org.apache.storm.topology.IWindowedBolt#execute(TupleWindow)}
+ * for the already evaluated windows in case of restarts during failures. The
+ * {@link org.apache.storm.topology.IStatefulBolt#initState(State)}
* is invoked with the previously saved state of the bolt after prepare, before the execute() method is invoked.
* </p>
*/
public class StatefulWindowingTopology {
private static final Logger LOG = LoggerFactory.getLogger(StatefulWindowingTopology.class);
+ public static void main(String[] args) throws Exception {
+ TopologyBuilder builder = new TopologyBuilder();
+ builder.setSpout("spout", new RandomIntegerSpout());
+ builder.setBolt("sumbolt", new WindowSumBolt().withWindow(new Count(5), new Count(3))
+ .withMessageIdField("msgid"), 1).shuffleGrouping("spout");
+ builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("sumbolt");
+ Config conf = new Config();
+ conf.setDebug(false);
+ //conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
+
+ String topoName = "test";
+ if (args != null && args.length > 0) {
+ topoName = args[0];
+ }
+ conf.setNumWorkers(1);
+ StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
+ }
+
private static class WindowSumBolt extends BaseStatefulWindowedBolt<KeyValueState<String, Long>> {
private KeyValueState<String, Long> state;
private long sum;
@@ -85,22 +103,4 @@ public class StatefulWindowingTopology {
}
}
- public static void main(String[] args) throws Exception {
- TopologyBuilder builder = new TopologyBuilder();
- builder.setSpout("spout", new RandomIntegerSpout());
- builder.setBolt("sumbolt", new WindowSumBolt().withWindow(new Count(5), new Count(3))
- .withMessageIdField("msgid"), 1).shuffleGrouping("spout");
- builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("sumbolt");
- Config conf = new Config();
- conf.setDebug(false);
- //conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
-
- String topoName = "test";
- if (args != null && args.length > 0) {
- topoName = args[0];
- }
- conf.setNumWorkers(1);
- StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
- }
-
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalGlobalCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalGlobalCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalGlobalCount.java
index 363fc2c..45cd68d 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalGlobalCount.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalGlobalCount.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import java.math.BigInteger;
@@ -22,7 +17,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.coordination.BatchOutputCollector;
@@ -74,15 +68,31 @@ public class TransactionalGlobalCount {
add(new Values("dog"));
}});
}};
+ public static final String GLOBAL_COUNT_KEY = "GLOBAL-COUNT";
+ public static Map<String, Value> DATABASE = new HashMap<String, Value>();
+
+ public static void main(String[] args) throws Exception {
+ if (!NimbusClient.isLocalOverride()) {
+ throw new IllegalStateException("This example only works in local mode. "
+ + "Run with storm local not storm jar");
+ }
+ MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
+ TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 3);
+ builder.setBolt("partial-count", new BatchCount(), 5).noneGrouping("spout");
+ builder.setBolt("sum", new UpdateGlobalCount()).globalGrouping("partial-count");
+
+ Config config = new Config();
+ config.setDebug(true);
+ config.setMaxSpoutPending(3);
+
+ StormSubmitter.submitTopology("global-count-topology", config, builder.buildTopology());
+ }
public static class Value {
int count = 0;
BigInteger txid;
}
- public static Map<String, Value> DATABASE = new HashMap<String, Value>();
- public static final String GLOBAL_COUNT_KEY = "GLOBAL-COUNT";
-
public static class BatchCount extends BaseBatchBolt<Object> {
Object _id;
BatchOutputCollector _collector;
@@ -137,13 +147,11 @@ public class TransactionalGlobalCount {
newval.txid = _attempt.getTransactionId();
if (val == null) {
newval.count = _sum;
- }
- else {
+ } else {
newval.count = _sum + val.count;
}
DATABASE.put(GLOBAL_COUNT_KEY, newval);
- }
- else {
+ } else {
newval = val;
}
_collector.emit(new Values(_attempt, newval.count));
@@ -154,21 +162,4 @@ public class TransactionalGlobalCount {
declarer.declare(new Fields("id", "sum"));
}
}
-
- public static void main(String[] args) throws Exception {
- if (!NimbusClient.isLocalOverride()) {
- throw new IllegalStateException("This example only works in local mode. "
- + "Run with storm local not storm jar");
- }
- MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
- TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 3);
- builder.setBolt("partial-count", new BatchCount(), 5).noneGrouping("spout");
- builder.setBolt("sum", new UpdateGlobalCount()).globalGrouping("partial-count");
-
- Config config = new Config();
- config.setDebug(true);
- config.setMaxSpoutPending(3);
-
- StormSubmitter.submitTopology("global-count-topology", config, builder.buildTopology());
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalWords.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalWords.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalWords.java
index 7e343af..7875360 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalWords.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/TransactionalWords.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import java.math.BigInteger;
@@ -22,7 +17,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.coordination.BatchOutputCollector;
@@ -51,25 +45,8 @@ import org.apache.storm.utils.NimbusClient;
* between buckets as their counts accumulate.
*/
public class TransactionalWords {
- public static class CountValue {
- Integer prev_count = null;
- int count = 0;
- BigInteger txid = null;
- }
-
- public static class BucketValue {
- int count = 0;
- BigInteger txid;
- }
-
public static final int BUCKET_SIZE = 10;
-
- public static Map<String, CountValue> COUNT_DATABASE = new HashMap<String, CountValue>();
- public static Map<Integer, BucketValue> BUCKET_DATABASE = new HashMap<Integer, BucketValue>();
-
-
public static final int PARTITION_TAKE_PER_BATCH = 3;
-
public static final Map<Integer, List<List<Object>>> DATA = new HashMap<Integer, List<List<Object>>>() {{
put(0, new ArrayList<List<Object>>() {{
add(new Values("cat"));
@@ -97,6 +74,36 @@ public class TransactionalWords {
add(new Values("dog"));
}});
}};
+ public static Map<String, CountValue> COUNT_DATABASE = new HashMap<String, CountValue>();
+ public static Map<Integer, BucketValue> BUCKET_DATABASE = new HashMap<Integer, BucketValue>();
+
+ public static void main(String[] args) throws Exception {
+ if (!NimbusClient.isLocalOverride()) {
+ throw new IllegalStateException("This example only works in local mode. "
+ + "Run with storm local not storm jar");
+ }
+ MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
+ TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2);
+ builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word"));
+ builder.setBolt("bucketize", new Bucketize()).noneGrouping("count");
+ builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket"));
+ Config config = new Config();
+ config.setDebug(true);
+ config.setMaxSpoutPending(3);
+
+ StormSubmitter.submitTopology("top-n-topology", config, builder.buildTopology());
+ }
+
+ public static class CountValue {
+ Integer prev_count = null;
+ int count = 0;
+ BigInteger txid = null;
+ }
+
+ public static class BucketValue {
+ int count = 0;
+ BigInteger txid;
+ }
public static class KeyedCountUpdater extends BaseTransactionalBolt implements ICommitter {
Map<String, Integer> _counts = new HashMap<String, Integer>();
@@ -115,8 +122,9 @@ public class TransactionalWords {
public void execute(Tuple tuple) {
String key = tuple.getString(1);
Integer curr = _counts.get(key);
- if (curr == null)
+ if (curr == null) {
curr = 0;
+ }
_counts.put(key, curr + 1);
}
@@ -134,8 +142,7 @@ public class TransactionalWords {
}
newVal.count = newVal.count + _counts.get(key);
COUNT_DATABASE.put(key, newVal);
- }
- else {
+ } else {
newVal = val;
}
_collector.emit(new Values(_id, key, newVal.count, newVal.prev_count));
@@ -163,8 +170,7 @@ public class TransactionalWords {
if (prevBucket == null) {
collector.emit(new Values(attempt, currBucket, 1));
- }
- else if (currBucket != prevBucket) {
+ } else if (currBucket != prevBucket) {
collector.emit(new Values(attempt, currBucket, 1));
collector.emit(new Values(attempt, prevBucket, -1));
}
@@ -194,8 +200,9 @@ public class TransactionalWords {
Integer bucket = tuple.getInteger(1);
Integer delta = tuple.getInteger(2);
Integer curr = _accum.get(bucket);
- if (curr == null)
+ if (curr == null) {
curr = 0;
+ }
_accum.put(bucket, curr + delta);
}
@@ -208,11 +215,11 @@ public class TransactionalWords {
newVal = new BucketValue();
newVal.txid = _attempt.getTransactionId();
newVal.count = _accum.get(bucket);
- if (currVal != null)
+ if (currVal != null) {
newVal.count += currVal.count;
+ }
BUCKET_DATABASE.put(bucket, newVal);
- }
- else {
+ } else {
newVal = currVal;
}
_collector.emit(new Values(_attempt, bucket, newVal.count));
@@ -224,21 +231,4 @@ public class TransactionalWords {
declarer.declare(new Fields("id", "bucket", "count"));
}
}
-
- public static void main(String[] args) throws Exception {
- if (!NimbusClient.isLocalOverride()) {
- throw new IllegalStateException("This example only works in local mode. "
- + "Run with storm local not storm jar");
- }
- MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
- TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2);
- builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word"));
- builder.setBolt("bucketize", new Bucketize()).noneGrouping("count");
- builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket"));
- Config config = new Config();
- config.setDebug(true);
- config.setMaxSpoutPending(3);
-
- StormSubmitter.submitTopology("top-n-topology", config, builder.buildTopology());
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java
index 59e3e4b..32ec822 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopology.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.storm.starter.spout.RandomSentenceSpout;
import org.apache.storm.task.ShellBolt;
import org.apache.storm.topology.BasicOutputCollector;
@@ -37,65 +31,66 @@ import org.apache.storm.tuple.Values;
* capabilities.
*/
public class WordCountTopology extends ConfigurableTopology {
- public static class SplitSentence extends ShellBolt implements IRichBolt {
-
- public SplitSentence() {
- super("python", "splitsentence.py");
+ public static void main(String[] args) throws Exception {
+ ConfigurableTopology.start(new WordCountTopology(), args);
}
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("word"));
- }
+ protected int run(String[] args) throws Exception {
- @Override
- public Map<String, Object> getComponentConfiguration() {
- return null;
- }
- }
-
- public static class WordCount extends BaseBasicBolt {
- Map<String, Integer> counts = new HashMap<String, Integer>();
-
- @Override
- public void execute(Tuple tuple, BasicOutputCollector collector) {
- String word = tuple.getString(0);
- Integer count = counts.get(word);
- if (count == null)
- count = 0;
- count++;
- counts.put(word, count);
- collector.emit(new Values(word, count));
- }
+ TopologyBuilder builder = new TopologyBuilder();
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("word", "count"));
- }
- }
+ builder.setSpout("spout", new RandomSentenceSpout(), 5);
- public static void main(String[] args) throws Exception {
- ConfigurableTopology.start(new WordCountTopology(), args);
- }
+ builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
+ builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));
- protected int run(String[] args) throws Exception {
+ conf.setDebug(true);
- TopologyBuilder builder = new TopologyBuilder();
+ String topologyName = "word-count";
- builder.setSpout("spout", new RandomSentenceSpout(), 5);
+ conf.setNumWorkers(3);
- builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
- builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));
+ if (args != null && args.length > 0) {
+ topologyName = args[0];
+ }
+ return submit(topologyName, conf, builder);
+ }
- conf.setDebug(true);
+ public static class SplitSentence extends ShellBolt implements IRichBolt {
- String topologyName = "word-count";
+ public SplitSentence() {
+ super("python", "splitsentence.py");
+ }
- conf.setNumWorkers(3);
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word"));
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return null;
+ }
+ }
- if (args != null && args.length > 0) {
- topologyName = args[0];
+ public static class WordCount extends BaseBasicBolt {
+ Map<String, Integer> counts = new HashMap<String, Integer>();
+
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ String word = tuple.getString(0);
+ Integer count = counts.get(word);
+ if (count == null) {
+ count = 0;
+ }
+ count++;
+ counts.put(word, count);
+ collector.emit(new Values(word, count));
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word", "count"));
+ }
}
- return submit(topologyName, conf, builder);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java
index 39f5932..6df7678 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/WordCountTopologyNode.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.spout.ShellSpout;
@@ -38,6 +32,25 @@ import org.apache.storm.tuple.Values;
* This topology demonstrates Storm's stream groupings and multilang capabilities.
*/
public class WordCountTopologyNode {
+ public static void main(String[] args) throws Exception {
+
+ TopologyBuilder builder = new TopologyBuilder();
+
+ builder.setSpout("spout", new RandomSentence(), 5);
+
+ builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
+ builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));
+
+ Config conf = new Config();
+ conf.setDebug(true);
+ String topoName = "word-count";
+ if (args != null && args.length > 0) {
+ topoName = args[0];
+ }
+ conf.setNumWorkers(3);
+ StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
+ }
+
public static class SplitSentence extends ShellBolt implements IRichBolt {
public SplitSentence() {
@@ -79,8 +92,9 @@ public class WordCountTopologyNode {
public void execute(Tuple tuple, BasicOutputCollector collector) {
String word = tuple.getString(0);
Integer count = counts.get(word);
- if (count == null)
+ if (count == null) {
count = 0;
+ }
count++;
counts.put(word, count);
collector.emit(new Values(word, count));
@@ -91,23 +105,4 @@ public class WordCountTopologyNode {
declarer.declare(new Fields("word", "count"));
}
}
-
- public static void main(String[] args) throws Exception {
-
- TopologyBuilder builder = new TopologyBuilder();
-
- builder.setSpout("spout", new RandomSentence(), 5);
-
- builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
- builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));
-
- Config conf = new Config();
- conf.setDebug(true);
- String topoName = "word-count";
- if (args != null && args.length > 0) {
- topoName = args[0];
- }
- conf.setNumWorkers(3);
- StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java
index 9cf9e79..b04196c 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/AbstractRankerBolt.java
@@ -1,23 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.bolt;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.log4j.Logger;
import org.apache.storm.Config;
+import org.apache.storm.starter.tools.Rankings;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
@@ -25,11 +24,6 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils;
-import org.apache.log4j.Logger;
-import org.apache.storm.starter.tools.Rankings;
-
-import java.util.HashMap;
-import java.util.Map;
/**
* This abstract bolt provides the basic behavior of bolts that rank objects according to their count.
@@ -40,71 +34,70 @@ import java.util.Map;
*/
public abstract class AbstractRankerBolt extends BaseBasicBolt {
- private static final long serialVersionUID = 4931640198501530202L;
- private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = 2;
- private static final int DEFAULT_COUNT = 10;
-
- private final int emitFrequencyInSeconds;
- private final int count;
- private final Rankings rankings;
-
- public AbstractRankerBolt() {
- this(DEFAULT_COUNT, DEFAULT_EMIT_FREQUENCY_IN_SECONDS);
- }
+ private static final long serialVersionUID = 4931640198501530202L;
+ private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = 2;
+ private static final int DEFAULT_COUNT = 10;
- public AbstractRankerBolt(int topN) {
- this(topN, DEFAULT_EMIT_FREQUENCY_IN_SECONDS);
- }
+ private final int emitFrequencyInSeconds;
+ private final int count;
+ private final Rankings rankings;
- public AbstractRankerBolt(int topN, int emitFrequencyInSeconds) {
- if (topN < 1) {
- throw new IllegalArgumentException("topN must be >= 1 (you requested " + topN + ")");
+ public AbstractRankerBolt() {
+ this(DEFAULT_COUNT, DEFAULT_EMIT_FREQUENCY_IN_SECONDS);
}
- if (emitFrequencyInSeconds < 1) {
- throw new IllegalArgumentException(
- "The emit frequency must be >= 1 seconds (you requested " + emitFrequencyInSeconds + " seconds)");
+
+ public AbstractRankerBolt(int topN) {
+ this(topN, DEFAULT_EMIT_FREQUENCY_IN_SECONDS);
}
- count = topN;
- this.emitFrequencyInSeconds = emitFrequencyInSeconds;
- rankings = new Rankings(count);
- }
- protected Rankings getRankings() {
- return rankings;
- }
+ public AbstractRankerBolt(int topN, int emitFrequencyInSeconds) {
+ if (topN < 1) {
+ throw new IllegalArgumentException("topN must be >= 1 (you requested " + topN + ")");
+ }
+ if (emitFrequencyInSeconds < 1) {
+ throw new IllegalArgumentException(
+ "The emit frequency must be >= 1 seconds (you requested " + emitFrequencyInSeconds + " seconds)");
+ }
+ count = topN;
+ this.emitFrequencyInSeconds = emitFrequencyInSeconds;
+ rankings = new Rankings(count);
+ }
- /**
- * This method functions as a template method (design pattern).
- */
- @Override
- public final void execute(Tuple tuple, BasicOutputCollector collector) {
- if (TupleUtils.isTick(tuple)) {
- getLogger().debug("Received tick tuple, triggering emit of current rankings");
- emitRankings(collector);
+ protected Rankings getRankings() {
+ return rankings;
}
- else {
- updateRankingsWithTuple(tuple);
+
+ /**
+ * This method functions as a template method (design pattern).
+ */
+ @Override
+ public final void execute(Tuple tuple, BasicOutputCollector collector) {
+ if (TupleUtils.isTick(tuple)) {
+ getLogger().debug("Received tick tuple, triggering emit of current rankings");
+ emitRankings(collector);
+ } else {
+ updateRankingsWithTuple(tuple);
+ }
}
- }
- abstract void updateRankingsWithTuple(Tuple tuple);
+ abstract void updateRankingsWithTuple(Tuple tuple);
- private void emitRankings(BasicOutputCollector collector) {
- collector.emit(new Values(rankings.copy()));
- getLogger().debug("Rankings: " + rankings);
- }
+ private void emitRankings(BasicOutputCollector collector) {
+ collector.emit(new Values(rankings.copy()));
+ getLogger().debug("Rankings: " + rankings);
+ }
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("rankings"));
- }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("rankings"));
+ }
- @Override
- public Map<String, Object> getComponentConfiguration() {
- Map<String, Object> conf = new HashMap<String, Object>();
- conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, emitFrequencyInSeconds);
- return conf;
- }
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ Map<String, Object> conf = new HashMap<String, Object>();
+ conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, emitFrequencyInSeconds);
+ return conf;
+ }
- abstract Logger getLogger();
+ abstract Logger getLogger();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java
index 6950bfb..a6a8b49 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/IntermediateRankingsBolt.java
@@ -1,26 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.bolt;
-import org.apache.storm.tuple.Tuple;
import org.apache.log4j.Logger;
import org.apache.storm.starter.tools.Rankable;
import org.apache.storm.starter.tools.RankableObjectWithFields;
+import org.apache.storm.tuple.Tuple;
/**
* This bolt ranks incoming objects by their count.
@@ -30,29 +25,29 @@ import org.apache.storm.starter.tools.RankableObjectWithFields;
*/
public final class IntermediateRankingsBolt extends AbstractRankerBolt {
- private static final long serialVersionUID = -1369800530256637409L;
- private static final Logger LOG = Logger.getLogger(IntermediateRankingsBolt.class);
+ private static final long serialVersionUID = -1369800530256637409L;
+ private static final Logger LOG = Logger.getLogger(IntermediateRankingsBolt.class);
- public IntermediateRankingsBolt() {
- super();
- }
+ public IntermediateRankingsBolt() {
+ super();
+ }
- public IntermediateRankingsBolt(int topN) {
- super(topN);
- }
+ public IntermediateRankingsBolt(int topN) {
+ super(topN);
+ }
- public IntermediateRankingsBolt(int topN, int emitFrequencyInSeconds) {
- super(topN, emitFrequencyInSeconds);
- }
+ public IntermediateRankingsBolt(int topN, int emitFrequencyInSeconds) {
+ super(topN, emitFrequencyInSeconds);
+ }
- @Override
- void updateRankingsWithTuple(Tuple tuple) {
- Rankable rankable = RankableObjectWithFields.from(tuple);
- super.getRankings().updateWith(rankable);
- }
+ @Override
+ void updateRankingsWithTuple(Tuple tuple) {
+ Rankable rankable = RankableObjectWithFields.from(tuple);
+ super.getRankings().updateWith(rankable);
+ }
- @Override
- Logger getLogger() {
- return LOG;
- }
+ @Override
+ Logger getLogger() {
+ return LOG;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java
index 993a937..8364222 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/PrinterBolt.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.bolt;
import org.apache.storm.topology.BasicOutputCollector;
@@ -25,13 +20,13 @@ import org.apache.storm.tuple.Tuple;
public class PrinterBolt extends BaseBasicBolt {
- @Override
- public void execute(Tuple tuple, BasicOutputCollector collector) {
- System.out.println(tuple);
- }
+ @Override
+ public void execute(Tuple tuple, BasicOutputCollector collector) {
+ System.out.println(tuple);
+ }
- @Override
- public void declareOutputFields(OutputFieldsDeclarer ofd) {
- }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer ofd) {
+ }
}
[20/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
Fixing stylecheck problemswith storm-server
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f1c0bcbe
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f1c0bcbe
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f1c0bcbe
Branch: refs/heads/master
Commit: f1c0bcbed16085699f5fa809afb7c58f9925ce68
Parents: 1d8a9b6
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 20:53:42 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 00:03:38 2018 -0400
----------------------------------------------------------------------
storm-server/pom.xml | 2 +-
.../java/org/apache/storm/DaemonConfig.java | 454 ++--
.../ILocalClusterTrackedTopologyAware.java | 25 +-
.../java/org/apache/storm/LocalCluster.java | 809 ++++---
.../main/java/org/apache/storm/LocalDRPC.java | 12 +-
.../java/org/apache/storm/ProcessSimulator.java | 34 +-
.../src/main/java/org/apache/storm/Testing.java | 330 ++-
.../storm/blobstore/BlobKeySequenceInfo.java | 35 +-
.../apache/storm/blobstore/BlobStoreUtils.java | 35 +-
.../storm/blobstore/BlobSynchronizer.java | 49 +-
.../storm/blobstore/FileBlobStoreImpl.java | 154 +-
.../storm/blobstore/KeySequenceNumber.java | 38 +-
.../storm/blobstore/LocalFsBlobStore.java | 84 +-
.../storm/blobstore/LocalFsBlobStoreFile.java | 36 +-
.../container/ResourceIsolationInterface.java | 18 +-
.../storm/container/cgroup/CgroupManager.java | 85 +-
.../daemon/drpc/BlockingOutstandingRequest.java | 6 +-
.../java/org/apache/storm/daemon/drpc/DRPC.java | 87 +-
.../apache/storm/daemon/drpc/DRPCThrift.java | 3 +-
.../storm/daemon/drpc/OutstandingRequest.java | 4 +-
.../storm/daemon/drpc/RequestFactory.java | 3 +-
.../storm/daemon/metrics/MetricsUtils.java | 1 -
.../reporters/ConsolePreparableReporter.java | 18 +-
.../reporters/CsvPreparableReporter.java | 18 +-
.../reporters/JmxPreparableReporter.java | 18 +-
.../metrics/reporters/PreparableReporter.java | 28 +-
.../org/apache/storm/daemon/nimbus/Nimbus.java | 2040 +++++++++---------
.../apache/storm/daemon/nimbus/TopoCache.java | 46 +-
.../storm/daemon/nimbus/TopologyActions.java | 23 +-
.../storm/daemon/nimbus/TopologyResources.java | 107 +-
.../daemon/nimbus/TopologyStateTransition.java | 23 +-
.../storm/daemon/supervisor/BasicContainer.java | 675 +++---
.../supervisor/BasicContainerLauncher.java | 32 +-
.../storm/daemon/supervisor/Container.java | 305 +--
.../daemon/supervisor/ContainerLauncher.java | 48 +-
.../supervisor/ContainerRecoveryException.java | 19 +-
.../DefaultUncaughtExceptionHandler.java | 20 +-
.../supervisor/EventManagerPushCallback.java | 19 +-
.../storm/daemon/supervisor/Killable.java | 27 +-
.../storm/daemon/supervisor/LocalContainer.java | 26 +-
.../supervisor/LocalContainerLauncher.java | 22 +-
.../daemon/supervisor/OnlyLatestExecutor.java | 9 +-
.../daemon/supervisor/ReadClusterState.java | 109 +-
.../daemon/supervisor/RunAsUserContainer.java | 25 +-
.../supervisor/RunAsUserContainerLauncher.java | 34 +-
.../apache/storm/daemon/supervisor/Slot.java | 863 ++++----
.../daemon/supervisor/StandaloneSupervisor.java | 30 +-
.../storm/daemon/supervisor/Supervisor.java | 184 +-
.../daemon/supervisor/SupervisorUtils.java | 59 +-
.../apache/storm/daemon/supervisor/UniFunc.java | 19 +-
.../timer/ReportWorkerHeartbeats.java | 19 +-
.../supervisor/timer/SupervisorHealthCheck.java | 1 -
.../supervisor/timer/SupervisorHeartbeat.java | 24 +-
.../timer/SynchronizeAssignments.java | 43 +-
.../org/apache/storm/event/EventManager.java | 19 +-
.../org/apache/storm/event/EventManagerImp.java | 30 +-
.../apache/storm/healthcheck/HealthChecker.java | 19 +-
.../apache/storm/localizer/AsyncLocalizer.java | 278 +--
.../storm/localizer/BlobChangingCallback.java | 14 +-
.../org/apache/storm/localizer/GoodToGo.java | 46 +-
.../apache/storm/localizer/LocalResource.java | 18 +-
.../storm/localizer/LocalizedResource.java | 93 +-
.../LocalizedResourceRetentionSet.java | 28 +-
.../storm/localizer/LocallyCachedBlob.java | 115 +-
.../localizer/LocallyCachedTopologyBlob.java | 164 +-
.../storm/localizer/PortAndAssignment.java | 18 +-
.../storm/logging/ThriftAccessLogger.java | 20 +-
.../logging/filters/AccessLoggingFilter.java | 21 +-
.../metric/ClusterMetricsConsumerExecutor.java | 32 +-
.../metric/LoggingClusterMetricsConsumer.java | 38 +-
.../storm/metric/StormMetricsRegistry.java | 29 +-
.../org/apache/storm/metric/api/DataPoint.java | 19 +-
.../metric/api/IClusterMetricsConsumer.java | 32 +-
.../org/apache/storm/metricstore/AggLevel.java | 21 +-
.../apache/storm/metricstore/FilterOptions.java | 79 +-
.../org/apache/storm/metricstore/Metric.java | 82 +-
.../storm/metricstore/MetricException.java | 19 +-
.../apache/storm/metricstore/MetricStore.java | 21 +-
.../storm/metricstore/MetricStoreConfig.java | 23 +-
.../metricstore/NimbusMetricProcessor.java | 21 +-
.../metricstore/WorkerMetricsProcessor.java | 19 +-
.../storm/metricstore/rocksdb/KeyType.java | 32 +-
.../metricstore/rocksdb/MetricsCleaner.java | 19 +-
.../rocksdb/ReadOnlyStringMetadataCache.java | 25 +-
.../storm/metricstore/rocksdb/RocksDbKey.java | 114 +-
.../rocksdb/RocksDbMetricsWriter.java | 35 +-
.../storm/metricstore/rocksdb/RocksDbStore.java | 50 +-
.../storm/metricstore/rocksdb/RocksDbValue.java | 25 +-
.../metricstore/rocksdb/StringMetadata.java | 19 +-
.../rocksdb/StringMetadataCache.java | 57 +-
.../rocksdb/WritableStringMetadataCache.java | 22 +-
.../nimbus/AssignmentDistributionService.java | 130 +-
.../storm/nimbus/DefaultTopologyValidator.java | 22 +-
.../org/apache/storm/nimbus/ILeaderElector.java | 21 +-
.../nimbus/ITopologyActionNotifierPlugin.java | 19 +-
.../apache/storm/nimbus/ITopologyValidator.java | 23 +-
.../IWorkerHeartbeatsRecoveryStrategy.java | 18 +-
.../storm/nimbus/LeaderListenerCallback.java | 58 +-
.../nimbus/NimbusHeartbeatsPressureTest.java | 124 +-
.../storm/nimbus/StrictTopologyValidator.java | 22 +-
...TimeOutWorkerHeartbeatsRecoveryStrategy.java | 20 +-
...WorkerHeartbeatsRecoveryStrategyFactory.java | 24 +-
.../storm/pacemaker/IServerMessageHandler.java | 19 +-
.../org/apache/storm/pacemaker/Pacemaker.java | 123 +-
.../apache/storm/pacemaker/PacemakerServer.java | 109 +-
.../pacemaker/codec/ThriftNettyServerCodec.java | 67 +-
.../org/apache/storm/scheduler/Cluster.java | 219 +-
.../org/apache/storm/scheduler/Component.java | 22 +-
.../storm/scheduler/DefaultScheduler.java | 9 +-
.../apache/storm/scheduler/EvenScheduler.java | 4 +-
.../apache/storm/scheduler/ExecutorDetails.java | 6 +-
.../org/apache/storm/scheduler/INimbus.java | 22 +-
.../org/apache/storm/scheduler/IScheduler.java | 22 +-
.../storm/scheduler/ISchedulingState.java | 118 +-
.../org/apache/storm/scheduler/ISupervisor.java | 21 +-
.../storm/scheduler/IsolationScheduler.java | 51 +-
.../storm/scheduler/SchedulerAssignment.java | 20 +-
.../scheduler/SchedulerAssignmentImpl.java | 35 +-
.../storm/scheduler/SingleTopologyCluster.java | 2 +-
.../storm/scheduler/SupervisorDetails.java | 51 +-
.../storm/scheduler/SupervisorResources.java | 4 +-
.../org/apache/storm/scheduler/Topologies.java | 30 +-
.../apache/storm/scheduler/TopologyDetails.java | 94 +-
.../scheduler/blacklist/BlacklistScheduler.java | 44 +-
.../blacklist/reporters/IReporter.java | 18 +-
.../blacklist/reporters/LogReporter.java | 21 +-
.../strategies/DefaultBlacklistStrategy.java | 29 +-
.../strategies/IBlacklistStrategy.java | 19 +-
.../scheduler/multitenant/DefaultPool.java | 363 ++--
.../storm/scheduler/multitenant/FreePool.java | 182 +-
.../scheduler/multitenant/IsolatedPool.java | 617 +++---
.../multitenant/MultitenantScheduler.java | 186 +-
.../storm/scheduler/multitenant/Node.java | 561 +++--
.../storm/scheduler/multitenant/NodePool.java | 494 +++--
.../storm/scheduler/resource/RAS_Node.java | 101 +-
.../storm/scheduler/resource/RAS_Nodes.java | 8 +-
.../resource/ResourceAwareScheduler.java | 90 +-
.../storm/scheduler/resource/ResourceUtils.java | 34 +-
.../scheduler/resource/SchedulingResult.java | 20 +-
.../scheduler/resource/SchedulingStatus.java | 20 +-
.../apache/storm/scheduler/resource/User.java | 10 +-
.../normalization/NormalizedResourceOffer.java | 9 +-
.../NormalizedResourceRequest.java | 54 +-
.../normalization/NormalizedResources.java | 66 +-
.../normalization/ResourceNameNormalizer.java | 10 +-
.../DefaultSchedulingPriorityStrategy.java | 90 +-
.../FIFOSchedulingPriorityStrategy.java | 28 +-
.../priority/ISchedulingPriorityStrategy.java | 19 +-
.../scheduling/BaseResourceAwareStrategy.java | 231 +-
.../scheduling/ConstraintSolverStrategy.java | 709 +++---
.../DefaultResourceAwareStrategy.java | 93 +-
.../GenericResourceAwareStrategy.java | 129 +-
.../strategies/scheduling/IStrategy.java | 23 +-
.../utils/ArtifactoryConfigLoader.java | 115 +-
.../utils/ArtifactoryConfigLoaderFactory.java | 18 +-
.../utils/ConfigLoaderFactoryService.java | 18 +-
.../storm/scheduler/utils/FileConfigLoader.java | 19 +-
.../utils/FileConfigLoaderFactory.java | 18 +-
.../storm/scheduler/utils/IConfigLoader.java | 20 +-
.../scheduler/utils/IConfigLoaderFactory.java | 18 +-
.../auth/workertoken/WorkerTokenManager.java | 64 +-
.../storm/testing/CompleteTopologyParam.java | 146 +-
.../storm/testing/InProcessZookeeper.java | 27 +-
.../java/org/apache/storm/testing/TestJob.java | 31 +-
.../apache/storm/testing/TrackedTopology.java | 164 +-
.../apache/storm/utils/BufferInputStream.java | 18 +-
.../java/org/apache/storm/utils/LruMap.java | 20 +-
.../apache/storm/utils/ServerConfigUtils.java | 16 +-
.../org/apache/storm/utils/ServerUtils.java | 201 +-
.../storm/utils/StormCommonInstaller.java | 20 +-
.../apache/storm/zookeeper/AclEnforcement.java | 42 +-
.../storm/zookeeper/LeaderElectorImp.java | 30 +-
.../org/apache/storm/zookeeper/Zookeeper.java | 54 +-
.../java/org/apache/storm/DaemonConfigTest.java | 42 +-
.../java/org/apache/storm/LocalStateTest.java | 38 +-
.../java/org/apache/storm/MessagingTest.java | 38 +-
.../java/org/apache/storm/PacemakerTest.java | 25 +-
.../test/java/org/apache/storm/TestCgroups.java | 51 +-
.../apache/storm/TestDaemonConfigValidate.java | 24 +-
.../java/org/apache/storm/TestRebalance.java | 47 +-
.../test/java/org/apache/storm/TestingTest.java | 50 +-
.../java/org/apache/storm/TickTupleTest.java | 41 +-
.../apache/storm/blobstore/BlobStoreTest.java | 798 ++++---
.../storm/blobstore/BlobStoreUtilsTest.java | 34 +-
.../storm/blobstore/BlobSynchronizerTest.java | 201 +-
.../blobstore/MockZookeeperClientBuilder.java | 30 +-
.../org/apache/storm/daemon/drpc/DRPCTest.java | 60 +-
.../apache/storm/daemon/nimbus/NimbusTest.java | 3 +-
.../daemon/supervisor/BasicContainerTest.java | 776 +++----
.../storm/daemon/supervisor/ContainerTest.java | 231 +-
.../storm/daemon/supervisor/SlotTest.java | 259 +--
.../storm/localizer/AsyncLocalizerTest.java | 342 ++-
.../LocalizedResourceRetentionSetTest.java | 31 +-
.../ClusterMetricsConsumerExecutorTest.java | 73 +-
.../apache/storm/nimbus/LocalNimbusTest.java | 36 +-
.../org/apache/storm/scheduler/ClusterTest.java | 24 +-
.../resource/TestResourceAwareScheduler.java | 99 +-
.../storm/scheduler/resource/TestUser.java | 40 +-
.../TestUtilsForResourceAwareScheduler.java | 88 +-
.../utils/ArtifactoryConfigLoaderTest.java | 114 +-
.../scheduler/utils/FileConfigLoaderTest.java | 29 +-
.../apache/storm/security/auth/AuthTest.java | 356 +--
202 files changed, 9484 insertions(+), 10315 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/pom.xml
----------------------------------------------------------------------
diff --git a/storm-server/pom.xml b/storm-server/pom.xml
index 9ff7f1d..236b4f1 100644
--- a/storm-server/pom.xml
+++ b/storm-server/pom.xml
@@ -138,7 +138,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>2585</maxAllowedViolations>
+ <maxAllowedViolations>783</maxAllowedViolations>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/DaemonConfig.java b/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
index be26ece..8fda533 100644
--- a/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
+++ b/storm-server/src/main/java/org/apache/storm/DaemonConfig.java
@@ -18,23 +18,8 @@
package org.apache.storm;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isBoolean;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isInteger;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isListEntryCustom;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isPositiveNumber;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isString;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isStringList;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isStringOrStringList;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isNumber;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isImplementationOfClass;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isMapEntryType;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isNoDuplicateInList;
-import static org.apache.storm.validation.ConfigValidationAnnotations.isMapEntryCustom;
-import static org.apache.storm.validation.ConfigValidationAnnotations.NotNull;
-
import java.util.ArrayList;
import java.util.Map;
-
import org.apache.storm.container.ResourceIsolationInterface;
import org.apache.storm.nimbus.ITopologyActionNotifierPlugin;
import org.apache.storm.scheduler.blacklist.reporters.IReporter;
@@ -45,11 +30,24 @@ import org.apache.storm.security.auth.IAuthorizer;
import org.apache.storm.validation.ConfigValidation;
import org.apache.storm.validation.Validated;
+import static org.apache.storm.validation.ConfigValidationAnnotations.NotNull;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isBoolean;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isImplementationOfClass;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isInteger;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isListEntryCustom;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isMapEntryCustom;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isMapEntryType;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isNoDuplicateInList;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isNumber;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isPositiveNumber;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isString;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isStringList;
+import static org.apache.storm.validation.ConfigValidationAnnotations.isStringOrStringList;
+
/**
- * Storm configs are specified as a plain old map. This class provides constants for
- * all the configurations possible on a Storm cluster. Each constant is paired with an annotation
- * that defines the validity criterion of the corresponding field. Default
- * values for these configs can be found in defaults.yaml.
+ * Storm configs are specified as a plain old map. This class provides constants for all the configurations possible on a Storm cluster.
+ * Each constant is paired with an annotation that defines the validity criterion of the corresponding field. Default values for these
+ * configs can be found in defaults.yaml.
*
* This class extends {@link org.apache.storm.Config} for supporting Storm Daemons.
*/
@@ -62,8 +60,8 @@ public class DaemonConfig implements Validated {
public static final String STORM_NETTY_FLUSH_CHECK_INTERVAL_MS = "storm.messaging.netty.flush.check.interval.ms";
/**
- * A list of daemon metrics reporter plugin class names.
- * These plugins must implement {@link org.apache.storm.daemon.metrics.reporters.PreparableReporter} interface.
+ * A list of daemon metrics reporter plugin class names. These plugins must implement {@link
+ * org.apache.storm.daemon.metrics.reporters.PreparableReporter} interface.
*/
@isStringList
public static final String STORM_DAEMON_METRICS_REPORTER_PLUGINS = "storm.daemon.metrics.reporter.plugins";
@@ -81,9 +79,8 @@ public class DaemonConfig implements Validated {
public static final String STORM_DAEMON_METRICS_REPORTER_CSV_LOG_DIR = "storm.daemon.metrics.reporter.csv.log.dir";
/**
- * A directory that holds configuration files for log4j2.
- * It can be either a relative or an absolute directory.
- * If relative, it is relative to the storm's home directory.
+ * A directory that holds configuration files for log4j2. It can be either a relative or an absolute directory. If relative, it is
+ * relative to the storm's home directory.
*/
@isString
public static final String STORM_LOG4J2_CONF_DIR = "storm.log4j2.conf.dir";
@@ -129,8 +126,8 @@ public class DaemonConfig implements Validated {
public static final String BLACKLIST_SCHEDULER_STRATEGY = "blacklist.scheduler.strategy";
/**
- * Whether we want to display all the resource capacity and scheduled usage on the UI page.
- * You MUST have this variable set if you are using any kind of resource-related scheduler.
+ * Whether we want to display all the resource capacity and scheduled usage on the UI page. You MUST have this variable set if you are
+ * using any kind of resource-related scheduler.
* <p/>
* If this is not set, we will not display resource capacity and usage on the UI.
*/
@@ -144,30 +141,27 @@ public class DaemonConfig implements Validated {
public static final String STORM_HEALTH_CHECK_DIR = "storm.health.check.dir";
/**
- * The time to allow any given healthcheck script to run before it
- * is marked failed due to timeout.
+ * The time to allow any given healthcheck script to run before it is marked failed due to timeout.
*/
@isNumber
public static final String STORM_HEALTH_CHECK_TIMEOUT_MS = "storm.health.check.timeout.ms";
/**
- * This is the user that the Nimbus daemon process is running as. May be used when security
- * is enabled to authorize actions in the cluster.
+ * This is the user that the Nimbus daemon process is running as. May be used when security is enabled to authorize actions in the
+ * cluster.
*/
@isString
public static final String NIMBUS_DAEMON_USER = "nimbus.daemon.user";
/**
- * This parameter is used by the storm-deploy project to configure the
- * jvm options for the nimbus daemon.
+ * This parameter is used by the storm-deploy project to configure the jvm options for the nimbus daemon.
*/
@isStringOrStringList
public static final String NIMBUS_CHILDOPTS = "nimbus.childopts";
/**
- * How long without heartbeating a task can go before nimbus will consider the
- * task dead and reassign it to another location.
+ * How long without heartbeating a task can go before nimbus will consider the task dead and reassign it to another location.
*/
@isInteger
@isPositiveNumber
@@ -175,10 +169,8 @@ public class DaemonConfig implements Validated {
/**
- * How often nimbus should wake up to check heartbeats and do reassignments. Note
- * that if a machine ever goes down Nimbus will immediately wake up and take action.
- * This parameter is for checking for failures when there's no explicit event like that
- * occurring.
+ * How often nimbus should wake up to check heartbeats and do reassignments. Note that if a machine ever goes down Nimbus will
+ * immediately wake up and take action. This parameter is for checking for failures when there's no explicit event like that occurring.
*/
@isInteger
@isPositiveNumber
@@ -186,6 +178,7 @@ public class DaemonConfig implements Validated {
/**
* How often nimbus should wake the cleanup thread to clean the inbox.
+ *
* @see #NIMBUS_INBOX_JAR_EXPIRATION_SECS
*/
@isInteger
@@ -195,26 +188,25 @@ public class DaemonConfig implements Validated {
/**
* The length of time a jar file lives in the inbox before being deleted by the cleanup thread.
*
- * Probably keep this value greater than or equal to NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS.
- * Note that the time it takes to delete an inbox jar file is going to be somewhat more than
- * NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS (depending on how often NIMBUS_CLEANUP_FREQ_SECS
- * is set to).
+ * Probably keep this value greater than or equal to NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS. Note that the time it takes to delete an
+ * inbox jar file is going to be somewhat more than NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS (depending on how often
+ * NIMBUS_CLEANUP_FREQ_SECS is set to).
+ *
* @see #NIMBUS_CLEANUP_INBOX_FREQ_SECS
*/
@isInteger
public static final String NIMBUS_INBOX_JAR_EXPIRATION_SECS = "nimbus.inbox.jar.expiration.secs";
/**
- * How long before a supervisor can go without heartbeating before nimbus considers it dead
- * and stops assigning new work to it.
+ * How long before a supervisor can go without heartbeating before nimbus considers it dead and stops assigning new work to it.
*/
@isInteger
@isPositiveNumber
public static final String NIMBUS_SUPERVISOR_TIMEOUT_SECS = "nimbus.supervisor.timeout.secs";
/**
- * A special timeout used when a task is initially launched. During launch, this is the timeout
- * used until the first heartbeat, overriding nimbus.task.timeout.secs.
+ * A special timeout used when a task is initially launched. During launch, this is the timeout used until the first heartbeat,
+ * overriding nimbus.task.timeout.secs.
*
* <p>A separate timeout exists for launch because there can be quite a bit of overhead
* to launching new JVM's and configuring them.</p>
@@ -224,16 +216,15 @@ public class DaemonConfig implements Validated {
public static final String NIMBUS_TASK_LAUNCH_SECS = "nimbus.task.launch.secs";
/**
- * During upload/download with the master, how long an upload or download connection is idle
- * before nimbus considers it dead and drops the connection.
+ * During upload/download with the master, how long an upload or download connection is idle before nimbus considers it dead and drops
+ * the connection.
*/
@isInteger
public static final String NIMBUS_FILE_COPY_EXPIRATION_SECS = "nimbus.file.copy.expiration.secs";
/**
- * A custom class that implements ITopologyValidator that is run whenever a
- * topology is submitted. Can be used to provide business-specific logic for
- * whether topologies are allowed to run or not.
+ * A custom class that implements ITopologyValidator that is run whenever a topology is submitted. Can be used to provide
+ * business-specific logic for whether topologies are allowed to run or not.
*/
@isString
public static final String NIMBUS_TOPOLOGY_VALIDATOR = "nimbus.topology.validator";
@@ -314,8 +305,8 @@ public class DaemonConfig implements Validated {
public static final String UI_CENTRAL_LOGGING_URL = "ui.central.logging.url";
/**
- * Storm UI drop-down pagination value. Set ui.pagination to be a positive integer
- * or -1 (displays all entries). Valid values: -1, 10, 20, 25 etc.
+ * Storm UI drop-down pagination value. Set ui.pagination to be a positive integer or -1 (displays all entries). Valid values: -1, 10,
+ * 20, 25 etc.
*/
@isInteger
public static final String UI_PAGINATION = "ui.pagination";
@@ -379,8 +370,8 @@ public class DaemonConfig implements Validated {
public static final String LOGVIEWER_HTTPS_KEYSTORE_PASSWORD = "logviewer.https.keystore.password";
/**
- * Type of the keystore for HTTPS for Storm Logviewer.
- * see http://docs.oracle.com/javase/8/docs/api/java/security/KeyStore.html for more details.
+ * Type of the keystore for HTTPS for Storm Logviewer. see http://docs.oracle.com/javase/8/docs/api/java/security/KeyStore.html for more
+ * details.
*/
@isString
public static final String LOGVIEWER_HTTPS_KEYSTORE_TYPE = "logviewer.https.keystore.type";
@@ -404,8 +395,8 @@ public class DaemonConfig implements Validated {
public static final String LOGVIEWER_HTTPS_TRUSTSTORE_PASSWORD = "logviewer.https.truststore.password";
/**
- * Type of the truststore for HTTPS for Storm Logviewer.
- * see http://docs.oracle.com/javase/8/docs/api/java/security/Truststore.html for more details.
+ * Type of the truststore for HTTPS for Storm Logviewer. see http://docs.oracle.com/javase/8/docs/api/java/security/Truststore.html for
+ * more details.
*/
@isString
public static final String LOGVIEWER_HTTPS_TRUSTSTORE_TYPE = "logviewer.https.truststore.type";
@@ -482,8 +473,9 @@ public class DaemonConfig implements Validated {
public static final String UI_HTTPS_KEYSTORE_PASSWORD = "ui.https.keystore.password";
/**
- * Type of keystore used by Storm UI for setting up HTTPS (SSL).
- * see http://docs.oracle.com/javase/7/docs/api/java/security/KeyStore.html for more details.
+ * Type of keystore used by Storm UI for setting up HTTPS (SSL). see http://docs.oracle.com/javase/7/docs/api/java/security/KeyStore
+ * .html
+ * for more details.
*/
@isString
public static final String UI_HTTPS_KEYSTORE_TYPE = "ui.https.keystore.type";
@@ -507,8 +499,9 @@ public class DaemonConfig implements Validated {
public static final String UI_HTTPS_TRUSTSTORE_PASSWORD = "ui.https.truststore.password";
/**
- * Type of truststore used by Storm UI for setting up HTTPS (SSL).
- * see http://docs.oracle.com/javase/7/docs/api/java/security/KeyStore.html for more details.
+ * Type of truststore used by Storm UI for setting up HTTPS (SSL). see http://docs.oracle
+ * .com/javase/7/docs/api/java/security/KeyStore.html
+ * for more details.
*/
@isString
public static final String UI_HTTPS_TRUSTSTORE_TYPE = "ui.https.truststore.type";
@@ -523,17 +516,15 @@ public class DaemonConfig implements Validated {
public static final String UI_HTTPS_NEED_CLIENT_AUTH = "ui.https.need.client.auth";
/**
- * The maximum number of threads that should be used by the Pacemaker.
- * When Pacemaker gets loaded it will spawn new threads, up to
- * this many total, to handle the load.
+ * The maximum number of threads that should be used by the Pacemaker. When Pacemaker gets loaded it will spawn new threads, up to this
+ * many total, to handle the load.
*/
@isNumber
@isPositiveNumber
public static final String PACEMAKER_MAX_THREADS = "pacemaker.max.threads";
/**
- * This parameter is used by the storm-deploy project to configure the
- * jvm options for the pacemaker daemon.
+ * This parameter is used by the storm-deploy project to configure the jvm options for the pacemaker daemon.
*/
@isStringOrStringList
public static final String PACEMAKER_CHILDOPTS = "pacemaker.childopts";
@@ -564,8 +555,9 @@ public class DaemonConfig implements Validated {
public static final String DRPC_HTTPS_KEYSTORE_PASSWORD = "drpc.https.keystore.password";
/**
- * Type of keystore used by Storm DRPC for setting up HTTPS (SSL).
- * see http://docs.oracle.com/javase/7/docs/api/java/security/KeyStore.html for more details.
+ * Type of keystore used by Storm DRPC for setting up HTTPS (SSL). see http://docs.oracle
+ * .com/javase/7/docs/api/java/security/KeyStore.html
+ * for more details.
*/
@isString
public static final String DRPC_HTTPS_KEYSTORE_TYPE = "drpc.https.keystore.type";
@@ -589,8 +581,9 @@ public class DaemonConfig implements Validated {
public static final String DRPC_HTTPS_TRUSTSTORE_PASSWORD = "drpc.https.truststore.password";
/**
- * Type of truststore used by Storm DRPC for setting up HTTPS (SSL).
- * see http://docs.oracle.com/javase/7/docs/api/java/security/KeyStore.html for more details.
+ * Type of truststore used by Storm DRPC for setting up HTTPS (SSL). see http://docs.oracle
+ * .com/javase/7/docs/api/java/security/KeyStore.html
+ * for more details.
*/
@isString
public static final String DRPC_HTTPS_TRUSTSTORE_TYPE = "drpc.https.truststore.type";
@@ -611,15 +604,14 @@ public class DaemonConfig implements Validated {
public static final String DRPC_AUTHORIZER = "drpc.authorizer";
/**
- * The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can also
- * timeout based on the socket timeout on the DRPC client, and separately based on the topology message
- * timeout for the topology implementing the DRPC function.
+ * The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can also timeout based on the socket
+ * timeout on the DRPC client, and separately based on the topology message timeout for the topology implementing the DRPC function.
*/
@isInteger
@isPositiveNumber
@NotNull
- public static final String DRPC_REQUEST_TIMEOUT_SECS = "drpc.request.timeout.secs";
+ public static final String DRPC_REQUEST_TIMEOUT_SECS = "drpc.request.timeout.secs";
/**
* Childopts for Storm DRPC Java process.
@@ -634,13 +626,12 @@ public class DaemonConfig implements Validated {
public static final String SUPERVISOR_SCHEDULER_META = "supervisor.scheduler.meta";
/**
- * A list of ports that can run workers on this supervisor. Each worker uses one port, and
- * the supervisor will only run one worker per port. Use this configuration to tune
- * how many workers run on each machine.
+ * A list of ports that can run workers on this supervisor. Each worker uses one port, and the supervisor will only run one worker per
+ * port. Use this configuration to tune how many workers run on each machine.
*/
@isNoDuplicateInList
@NotNull
- @isListEntryCustom(entryValidatorClasses = {ConfigValidation.IntegerValidator.class,ConfigValidation.PositiveNumberValidator.class})
+ @isListEntryCustom(entryValidatorClasses = { ConfigValidation.IntegerValidator.class, ConfigValidation.PositiveNumberValidator.class })
public static final String SUPERVISOR_SLOTS_PORTS = "supervisor.slots.ports";
/**
@@ -650,16 +641,14 @@ public class DaemonConfig implements Validated {
public static final String SUPERVISOR_BLOBSTORE = "supervisor.blobstore.class";
/**
- * The distributed cache target size in MB. This is a soft limit to the size of the distributed
- * cache contents.
+ * The distributed cache target size in MB. This is a soft limit to the size of the distributed cache contents.
*/
@isPositiveNumber
@isInteger
public static final String SUPERVISOR_LOCALIZER_CACHE_TARGET_SIZE_MB = "supervisor.localizer.cache.target.size.mb";
/**
- * The distributed cache cleanup interval. Controls how often it scans to attempt to cleanup
- * anything over the cache target size.
+ * The distributed cache cleanup interval. Controls how often it scans to attempt to cleanup anything over the cache target size.
*/
@isPositiveNumber
@isInteger
@@ -686,9 +675,8 @@ public class DaemonConfig implements Validated {
public static final String NIMBUS_BLOBSTORE = "nimbus.blobstore.class";
/**
- * During operations with the blob store, via master, how long a connection
- * is idle before nimbus considers it dead and drops the session and any
- * associated connections.
+ * During operations with the blob store, via master, how long a connection is idle before nimbus considers it dead and drops the
+ * session and any associated connections.
*/
@isPositiveNumber
@isInteger
@@ -708,8 +696,7 @@ public class DaemonConfig implements Validated {
public static final String DRPC_HTTP_FILTER = "drpc.http.filter";
/**
- * Initialization parameters for the javax.servlet.Filter of the DRPC HTTP
- * service.
+ * Initialization parameters for the javax.servlet.Filter of the DRPC HTTP service.
*/
@isMapEntryType(keyType = String.class, valueType = String.class)
public static final String DRPC_HTTP_FILTER_PARAMS = "drpc.http.filter.params";
@@ -722,8 +709,7 @@ public class DaemonConfig implements Validated {
public static final String NIMBUS_EXECUTORS_PER_TOPOLOGY = "nimbus.executors.perTopology";
/**
- * This parameter is used by the storm-deploy project to configure the
- * jvm options for the supervisor daemon.
+ * This parameter is used by the storm-deploy project to configure the jvm options for the supervisor daemon.
*/
@isStringOrStringList
public static final String SUPERVISOR_CHILDOPTS = "supervisor.childopts";
@@ -736,10 +722,9 @@ public class DaemonConfig implements Validated {
public static final String SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS = "supervisor.worker.shutdown.sleep.secs";
/**
- * How long a worker can go without heartbeating during the initial launch before
- * the supervisor tries to restart the worker process. This value override
- * supervisor.worker.timeout.secs during launch because there is additional
- * overhead to starting and configuring the JVM on launch.
+ * How long a worker can go without heartbeating during the initial launch before the supervisor tries to restart the worker process.
+ * This value override supervisor.worker.timeout.secs during launch because there is additional overhead to starting and configuring the
+ * JVM on launch.
*/
@isInteger
@isPositiveNumber
@@ -747,9 +732,8 @@ public class DaemonConfig implements Validated {
public static final String SUPERVISOR_WORKER_START_TIMEOUT_SECS = "supervisor.worker.start.timeout.secs";
/**
- * Whether or not the supervisor should launch workers assigned to it. Defaults
- * to true -- and you should probably never change this value. This configuration
- * is used in the Storm unit tests.
+ * Whether or not the supervisor should launch workers assigned to it. Defaults to true -- and you should probably never change this
+ * value. This configuration is used in the Storm unit tests.
*/
@isBoolean
public static final String SUPERVISOR_ENABLE = "supervisor.enable";
@@ -762,16 +746,13 @@ public class DaemonConfig implements Validated {
/**
- * How often the supervisor checks the worker heartbeats to see if any of them
- * need to be restarted.
+ * How often the supervisor checks the worker heartbeats to see if any of them need to be restarted.
*/
@isInteger
@isPositiveNumber
public static final String SUPERVISOR_MONITOR_FREQUENCY_SECS = "supervisor.monitor.frequency.secs";
-
-
/**
* The jvm profiler opts provided to workers launched by this supervisor.
*/
@@ -779,29 +760,25 @@ public class DaemonConfig implements Validated {
public static final String WORKER_PROFILER_CHILDOPTS = "worker.profiler.childopts";
/**
- * Enable profiling of worker JVMs using Oracle's Java Flight Recorder.
- * Unlocking commercial features requires a special license from Oracle.
- * See http://www.oracle.com/technetwork/java/javase/terms/products/index.html
+ * Enable profiling of worker JVMs using Oracle's Java Flight Recorder. Unlocking commercial features requires a special license from
+ * Oracle. See http://www.oracle.com/technetwork/java/javase/terms/products/index.html
*/
@isBoolean
public static final String WORKER_PROFILER_ENABLED = "worker.profiler.enabled";
/**
- * The command launched supervisor with worker arguments
- * pid, action and [target_directory]
- * Where action is - start profile, stop profile, jstack, heapdump and kill against pid.
- *
+ * The command launched supervisor with worker arguments pid, action and [target_directory] Where action is - start profile, stop
+ * profile, jstack, heapdump and kill against pid.
*/
@isString
public static final String WORKER_PROFILER_COMMAND = "worker.profiler.command";
/**
- * A list of classes implementing IClusterMetricsConsumer (See storm.yaml.example for exact config format).
- * Each listed class will be routed cluster related metrics data.
- * Each listed class maps 1:1 to a ClusterMetricsConsumerExecutor and they're executed in Nimbus.
+ * A list of classes implementing IClusterMetricsConsumer (See storm.yaml.example for exact config format). Each listed class will be
+ * routed cluster related metrics data. Each listed class maps 1:1 to a ClusterMetricsConsumerExecutor and they're executed in Nimbus.
* Only consumers which run in leader Nimbus receives metrics data.
*/
- @isListEntryCustom(entryValidatorClasses = {ConfigValidation.ClusterMetricRegistryValidator.class})
+ @isListEntryCustom(entryValidatorClasses = { ConfigValidation.ClusterMetricRegistryValidator.class })
public static final String STORM_CLUSTER_METRICS_CONSUMER_REGISTER = "storm.cluster.metrics.consumer.register";
/**
@@ -810,7 +787,7 @@ public class DaemonConfig implements Validated {
@NotNull
@isPositiveNumber
public static final String STORM_CLUSTER_METRICS_CONSUMER_PUBLISH_INTERVAL_SECS =
- "storm.cluster.metrics.consumer.publish.interval.secs";
+ "storm.cluster.metrics.consumer.publish.interval.secs";
/**
* Enables user-first classpath. See topology.classpath.beginning.
@@ -819,75 +796,72 @@ public class DaemonConfig implements Validated {
public static final String STORM_TOPOLOGY_CLASSPATH_BEGINNING_ENABLED = "storm.topology.classpath.beginning.enabled";
/**
- * This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers)
- * for the java.library.path value. java.library.path tells the JVM where
- * to look for native libraries. It is necessary to set this config correctly since
- * Storm uses the ZeroMQ and JZMQ native libs.
+ * This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers) for the java.library.path value. java.library.path tells
+ * the JVM where to look for native libraries. It is necessary to set this config correctly since Storm uses the ZeroMQ and JZMQ native
+ * libs.
*/
@isString
public static final String JAVA_LIBRARY_PATH = "java.library.path";
/**
- * The path to use as the zookeeper dir when running a zookeeper server via
- * "storm dev-zookeeper". This zookeeper instance is only intended for development;
- * it is not a production grade zookeeper setup.
+ * The path to use as the zookeeper dir when running a zookeeper server via "storm dev-zookeeper". This zookeeper instance is only
+ * intended for development; it is not a production grade zookeeper setup.
*/
@isString
public static final String DEV_ZOOKEEPER_PATH = "dev.zookeeper.path";
/**
- * A map from topology name to the number of machines that should be dedicated for that topology. Set storm.scheduler
- * to org.apache.storm.scheduler.IsolationScheduler to make use of the isolation scheduler.
+ * A map from topology name to the number of machines that should be dedicated for that topology. Set storm.scheduler to
+ * org.apache.storm.scheduler.IsolationScheduler to make use of the isolation scheduler.
*/
@isMapEntryType(keyType = String.class, valueType = Number.class)
public static final String ISOLATION_SCHEDULER_MACHINES = "isolation.scheduler.machines";
/**
- * For ArtifactoryConfigLoader, this can either be a reference to an individual file in Artifactory or to a directory.
- * If it is a directory, the file with the largest lexographic name will be returned. Users need to add "artifactory+"
- * to the beginning of the real URI to use ArtifactoryConfigLoader.
- * For FileConfigLoader, this is the URI pointing to a file.
+ * For ArtifactoryConfigLoader, this can either be a reference to an individual file in Artifactory or to a directory. If it is a
+ * directory, the file with the largest lexographic name will be returned. Users need to add "artifactory+" to the beginning of the real
+ * URI to use ArtifactoryConfigLoader. For FileConfigLoader, this is the URI pointing to a file.
*/
@isString
public static final String SCHEDULER_CONFIG_LOADER_URI = "scheduler.config.loader.uri";
/**
- * It is the frequency at which the plugin will call out to artifactory instead of returning the most recently cached result.
- * Currently it's only used in ArtifactoryConfigLoader.
+ * It is the frequency at which the plugin will call out to artifactory instead of returning the most recently cached result. Currently
+ * it's only used in ArtifactoryConfigLoader.
*/
@isInteger
@isPositiveNumber
public static final String SCHEDULER_CONFIG_LOADER_POLLTIME_SECS = "scheduler.config.loader.polltime.secs";
/**
- * It is the amount of time an http connection to the artifactory server will wait before timing out.
- * Currently it's only used in ArtifactoryConfigLoader.
+ * It is the amount of time an http connection to the artifactory server will wait before timing out. Currently it's only used in
+ * ArtifactoryConfigLoader.
*/
@isInteger
@isPositiveNumber
public static final String SCHEDULER_CONFIG_LOADER_TIMEOUT_SECS = "scheduler.config.loader.timeout.secs";
/**
- * It is the part of the uri, configurable in Artifactory, which represents the top of the directory tree.
- * It's only used in ArtifactoryConfigLoader.
+ * It is the part of the uri, configurable in Artifactory, which represents the top of the directory tree. It's only used in
+ * ArtifactoryConfigLoader.
*/
@isString
- public static final String SCHEDULER_CONFIG_LOADER_ARTIFACTORY_BASE_DIRECTORY = "scheduler.config.loader.artifactory.base.directory";
+ public static final String SCHEDULER_CONFIG_LOADER_ARTIFACTORY_BASE_DIRECTORY = "scheduler.config.loader.artifactory.base.directory";
/**
- * A map from the user name to the number of machines that should that user is allowed to use. Set storm.scheduler
- * to org.apache.storm.scheduler.multitenant.MultitenantScheduler
+ * A map from the user name to the number of machines that should that user is allowed to use. Set storm.scheduler to
+ * org.apache.storm.scheduler.multitenant.MultitenantScheduler
*/
@isMapEntryType(keyType = String.class, valueType = Number.class)
public static final String MULTITENANT_SCHEDULER_USER_POOLS = "multitenant.scheduler.user.pools";
/**
- * A map of users to another map of the resource guarantees of the user. Used by Resource Aware Scheduler to ensure
- * per user resource guarantees.
+ * A map of users to another map of the resource guarantees of the user. Used by Resource Aware Scheduler to ensure per user resource
+ * guarantees.
*/
@isMapEntryCustom(
- keyValidatorClasses = {ConfigValidation.StringValidator.class},
- valueValidatorClasses = {ConfigValidation.UserResourcePoolEntryValidator.class})
+ keyValidatorClasses = { ConfigValidation.StringValidator.class },
+ valueValidatorClasses = { ConfigValidation.UserResourcePoolEntryValidator.class })
public static final String RESOURCE_AWARE_SCHEDULER_USER_POOLS = "resource.aware.scheduler.user.pools";
/**
@@ -934,192 +908,162 @@ public class DaemonConfig implements Validated {
public static final String STORM_CGROUP_HIERARCHY_NAME = "storm.cgroup.hierarchy.name";
/**
- * flag to determine whether to use a resource isolation plugin
- * Also determines whether the unit tests for cgroup runs.
- * If storm.resource.isolation.plugin.enable is set to false the unit tests for cgroups will not run
+ * flag to determine whether to use a resource isolation plugin Also determines whether the unit tests for cgroup runs. If
+ * storm.resource.isolation.plugin.enable is set to false the unit tests for cgroups will not run
*/
@isBoolean
public static final String STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE = "storm.resource.isolation.plugin.enable";
+ /**
+ * Class implementing MetricStore. Runs on Nimbus.
+ */
+ @NotNull
+ @isString
+ // Validating class implementation could fail on non-Nimbus Daemons. Nimbus will catch the class not found on startup
+ // and log an error message, so just validating this as a String for now.
+ public static final String STORM_METRIC_STORE_CLASS = "storm.metricstore.class";
+ /**
+ * Class implementing WorkerMetricsProcessor. Runs on Supervisors.
+ */
+ @NotNull
+ @isString
+ public static final String STORM_METRIC_PROCESSOR_CLASS = "storm.metricprocessor.class";
+ /**
+ * RocksDB file location. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore implementation for the
+ * storm.metricstore.class.
+ */
+ @isString
+ public static final String STORM_ROCKSDB_LOCATION = "storm.metricstore.rocksdb.location";
+ /**
+ * RocksDB create if missing flag. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore implementation for
+ * the storm.metricstore.class.
+ */
+ @isBoolean
+ public static final String STORM_ROCKSDB_CREATE_IF_MISSING = "storm.metricstore.rocksdb.create_if_missing";
+ /**
+ * RocksDB metadata cache capacity. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore implementation for
+ * the storm.metricstore.class.
+ */
+ @isInteger
+ public static final String STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY = "storm.metricstore.rocksdb.metadata_string_cache_capacity";
+ /**
+ * RocksDB setting for length of metric retention. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore
+ * implementation for the storm.metricstore.class.
+ */
+ @isInteger
+ public static final String STORM_ROCKSDB_METRIC_RETENTION_HOURS = "storm.metricstore.rocksdb.retention_hours";
+ // Configs for memory enforcement done by the supervisor (not cgroups directly)
+ /**
+ * RocksDB setting for period of metric deletion thread. This setting is specific to the org.apache.storm.metricstore.rocksdb
+ * .RocksDbStore
+ * implementation for the storm.metricstore.class.
+ */
+ @isInteger
+ public static final String STORM_ROCKSDB_METRIC_DELETION_PERIOD_HOURS = "storm.metricstore.rocksdb.deletion_period_hours";
+ /**
+ * In nimbus on startup check if all of the zookeeper ACLs are correct before starting. If not don't start nimbus.
+ */
+ @isBoolean
+ public static final String STORM_NIMBUS_ZOOKEEPER_ACLS_CHECK = "storm.nimbus.zookeeper.acls.check";
+ /**
+ * In nimbus on startup check if all of the zookeeper ACLs are correct before starting. If not do your best to fix them before nimbus
+ * starts, if it cannot fix them nimbus will not start. This overrides any value set for storm.nimbus.zookeeper.acls.check.
+ */
+ @isBoolean
+ public static final String STORM_NIMBUS_ZOOKEEPER_ACLS_FIXUP = "storm.nimbus.zookeeper.acls.fixup";
+ /**
+ * Server side validation that @{see Config#TOPOLOGY_SCHEDULER_STRATEGY} is set ot a subclass of IStrategy.
+ */
+ @isImplementationOfClass(implementsClass = IStrategy.class)
+ public static final String VALIDATE_TOPOLOGY_SCHEDULER_STRATEGY = Config.TOPOLOGY_SCHEDULER_STRATEGY;
/**
* root directory for cgoups.
*/
@isString
public static String STORM_SUPERVISOR_CGROUP_ROOTDIR = "storm.supervisor.cgroup.rootdir";
-
/**
* the manually set memory limit (in MB) for each CGroup on supervisor node.
*/
@isPositiveNumber
public static String STORM_WORKER_CGROUP_MEMORY_MB_LIMIT = "storm.worker.cgroup.memory.mb.limit";
-
/**
* the manually set cpu share for each CGroup on supervisor node.
*/
@isPositiveNumber
public static String STORM_WORKER_CGROUP_CPU_LIMIT = "storm.worker.cgroup.cpu.limit";
-
/**
* full path to cgexec command.
*/
@isString
public static String STORM_CGROUP_CGEXEC_CMD = "storm.cgroup.cgexec.cmd";
-
/**
- * Please use STORM_SUPERVISOR_MEMORY_LIMIT_TOLERANCE_MARGIN_MB instead. The amount of memory a
- * worker can exceed its allocation before cgroup will kill it.
+ * Please use STORM_SUPERVISOR_MEMORY_LIMIT_TOLERANCE_MARGIN_MB instead. The amount of memory a worker can exceed its allocation before
+ * cgroup will kill it.
*/
@isPositiveNumber(includeZero = true)
public static String STORM_CGROUP_MEMORY_LIMIT_TOLERANCE_MARGIN_MB =
"storm.cgroup.memory.limit.tolerance.margin.mb";
-
/**
- * Java does not always play nicely with cgroups. It is coming but not fully implemented and not
- * for the way storm uses cgroups. In the short term you can disable the hard memory enforcement
- * by cgroups and let the supervisor handle shooting workers going over their limit in a kinder
- * way.
+ * Java does not always play nicely with cgroups. It is coming but not fully implemented and not for the way storm uses cgroups. In the
+ * short term you can disable the hard memory enforcement by cgroups and let the supervisor handle shooting workers going over their
+ * limit in a kinder way.
*/
@isBoolean
public static String STORM_CGROUP_MEMORY_ENFORCEMENT_ENABLE = "storm.cgroup.memory.enforcement.enable";
-
- // Configs for memory enforcement done by the supervisor (not cgroups directly)
-
/**
- * Memory given to each worker for free (because java and storm have some overhead). This is
- * memory on the box that the workers can use. This should not be included in
- * SUPERVISOR_MEMORY_CAPACITY_MB, as nimbus does not use this memory for scheduling.
+ * Memory given to each worker for free (because java and storm have some overhead). This is memory on the box that the workers can use.
+ * This should not be included in SUPERVISOR_MEMORY_CAPACITY_MB, as nimbus does not use this memory for scheduling.
*/
@isPositiveNumber
public static String STORM_SUPERVISOR_MEMORY_LIMIT_TOLERANCE_MARGIN_MB =
"storm.supervisor.memory.limit.tolerance.margin.mb";
-
/**
- * A multiplier for the memory limit of a worker that will have the supervisor shoot it
- * immediately. 1.0 means shoot the worker as soon as it goes over. 2.0 means shoot the worker if
- * its usage is double what was requested. This value is combined with
- * STORM_SUPERVISOR_HARD_MEMORY_LIMIT_OVERAGE and which ever is greater is used for enforcement.
- * This allows small workers to not be shot.
+ * A multiplier for the memory limit of a worker that will have the supervisor shoot it immediately. 1.0 means shoot the worker as soon
+ * as it goes over. 2.0 means shoot the worker if its usage is double what was requested. This value is combined with
+ * STORM_SUPERVISOR_HARD_MEMORY_LIMIT_OVERAGE and which ever is greater is used for enforcement. This allows small workers to not be
+ * shot.
*/
@isPositiveNumber
public static String STORM_SUPERVISOR_HARD_MEMORY_LIMIT_MULTIPLIER =
"storm.supervisor.hard.memory.limit.multiplier";
-
/**
- * If the memory usage of a worker goes over its limit by this value is it shot immediately. This
- * value is combined with STORM_SUPERVISOR_HARD_LIMIT_MEMORY_MULTIPLIER and which ever is greater
- * is used for enforcement. This allows small workers to not be shot.
+ * If the memory usage of a worker goes over its limit by this value is it shot immediately. This value is combined with
+ * STORM_SUPERVISOR_HARD_LIMIT_MEMORY_MULTIPLIER and which ever is greater is used for enforcement. This allows small workers to not be
+ * shot.
*/
@isPositiveNumber(includeZero = true)
public static String STORM_SUPERVISOR_HARD_LIMIT_MEMORY_OVERAGE_MB = "storm.supervisor.hard.memory.limit.overage.mb";
-
/**
- * If the amount of memory that is free in the system (either on the box or in the supervisor's
- * cgroup) is below this number (in MB) consider the system to be in low memory mode and start
- * shooting workers if they are over their limit.
+ * If the amount of memory that is free in the system (either on the box or in the supervisor's cgroup) is below this number (in MB)
+ * consider the system to be in low memory mode and start shooting workers if they are over their limit.
*/
@isPositiveNumber
public static String STORM_SUPERVISOR_LOW_MEMORY_THRESHOLD_MB = "storm.supervisor.low.memory.threshold.mb";
-
/**
- * If the amount of memory that is free in the system (either on the box or in the supervisor's
- * cgroup) is below this number (in MB) consider the system to be a little low on memory and start
- * shooting workers if they are over their limit for a given grace period
+ * If the amount of memory that is free in the system (either on the box or in the supervisor's cgroup) is below this number (in MB)
+ * consider the system to be a little low on memory and start shooting workers if they are over their limit for a given grace period
* STORM_SUPERVISOR_MEDIUM_MEMORY_GRACE_PERIOD_MS.
*/
@isPositiveNumber
public static String STORM_SUPERVISOR_MEDIUM_MEMORY_THRESHOLD_MB = "storm.supervisor.medium.memory.threshold.mb";
-
/**
- * The number of milliseconds that a worker is allowed to be over their limit when there is a
- * medium amount of memory free in the system.
+ * The number of milliseconds that a worker is allowed to be over their limit when there is a medium amount of memory free in the
+ * system.
*/
@isPositiveNumber
public static String STORM_SUPERVISOR_MEDIUM_MEMORY_GRACE_PERIOD_MS =
"storm.supervisor.medium.memory.grace.period.ms";
- /**
- * Class implementing MetricStore. Runs on Nimbus.
- */
- @NotNull
- @isString
- // Validating class implementation could fail on non-Nimbus Daemons. Nimbus will catch the class not found on startup
- // and log an error message, so just validating this as a String for now.
- public static final String STORM_METRIC_STORE_CLASS = "storm.metricstore.class";
-
- /**
- * Class implementing WorkerMetricsProcessor. Runs on Supervisors.
- */
- @NotNull
- @isString
- public static final String STORM_METRIC_PROCESSOR_CLASS = "storm.metricprocessor.class";
-
- /**
- * RocksDB file location. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore
- * implementation for the storm.metricstore.class.
- */
- @isString
- public static final String STORM_ROCKSDB_LOCATION = "storm.metricstore.rocksdb.location";
-
- /**
- * RocksDB create if missing flag. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore
- * implementation for the storm.metricstore.class.
- */
- @isBoolean
- public static final String STORM_ROCKSDB_CREATE_IF_MISSING = "storm.metricstore.rocksdb.create_if_missing";
-
- /**
- * RocksDB metadata cache capacity. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore
- * implementation for the storm.metricstore.class.
- */
- @isInteger
- public static final String STORM_ROCKSDB_METADATA_STRING_CACHE_CAPACITY = "storm.metricstore.rocksdb.metadata_string_cache_capacity";
-
- /**
- * RocksDB setting for length of metric retention. This setting is specific to the org.apache.storm.metricstore.rocksdb.RocksDbStore
- * implementation for the storm.metricstore.class.
- */
- @isInteger
- public static final String STORM_ROCKSDB_METRIC_RETENTION_HOURS = "storm.metricstore.rocksdb.retention_hours";
-
- /**
- * RocksDB setting for period of metric deletion thread. This setting is specific to the
- * org.apache.storm.metricstore.rocksdb.RocksDbStore implementation for the storm.metricstore.class.
- */
- @isInteger
- public static final String STORM_ROCKSDB_METRIC_DELETION_PERIOD_HOURS = "storm.metricstore.rocksdb.deletion_period_hours";
-
- /**
- * The number of hours a worker token is valid for. This also sets how frequently worker tokens will be renewed.
- */
- @isPositiveNumber
- public static String STORM_WORKER_TOKEN_LIFE_TIME_HOURS = "storm.worker.token.life.time.hours";
-
- /**
- * In nimbus on startup check if all of the zookeeper ACLs are correct before starting. If not
- * don't start nimbus.
- */
- @isBoolean
- public static final String STORM_NIMBUS_ZOOKEEPER_ACLS_CHECK = "storm.nimbus.zookeeper.acls.check";
-
- /**
- * In nimbus on startup check if all of the zookeeper ACLs are correct before starting. If not do
- * your best to fix them before nimbus starts, if it cannot fix them nimbus will not start.
- * This overrides any value set for storm.nimbus.zookeeper.acls.check.
- */
- @isBoolean
- public static final String STORM_NIMBUS_ZOOKEEPER_ACLS_FIXUP = "storm.nimbus.zookeeper.acls.fixup";
-
// VALIDATION ONLY CONFIGS
// Some configs inside Config.java may reference classes we don't want to expose in storm-client, but we still want to validate
// That they reference a valid class. To allow this to happen we do part of the validation on the client side with annotations on
// static final members of the Config class, and other validations here. We avoid naming them the same thing because clojure code
// walks these two classes and creates clojure constants for these values.
-
/**
- * Server side validation that @{see Config#TOPOLOGY_SCHEDULER_STRATEGY} is set ot a subclass of IStrategy.
+ * The number of hours a worker token is valid for. This also sets how frequently worker tokens will be renewed.
*/
- @isImplementationOfClass(implementsClass = IStrategy.class)
- public static final String VALIDATE_TOPOLOGY_SCHEDULER_STRATEGY = Config.TOPOLOGY_SCHEDULER_STRATEGY;
+ @isPositiveNumber
+ public static String STORM_WORKER_TOKEN_LIFE_TIME_HOURS = "storm.worker.token.life.time.hours";
public static String getCgroupRootDir(Map<String, Object> conf) {
return (String) conf.get(STORM_SUPERVISOR_CGROUP_ROOTDIR);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/ILocalClusterTrackedTopologyAware.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/ILocalClusterTrackedTopologyAware.java b/storm-server/src/main/java/org/apache/storm/ILocalClusterTrackedTopologyAware.java
index c67d32d..b279a8d 100644
--- a/storm-server/src/main/java/org/apache/storm/ILocalClusterTrackedTopologyAware.java
+++ b/storm-server/src/main/java/org/apache/storm/ILocalClusterTrackedTopologyAware.java
@@ -18,37 +18,40 @@
package org.apache.storm;
+import java.util.Map;
import org.apache.storm.generated.SubmitOptions;
import org.apache.storm.testing.TrackedTopology;
import org.apache.thrift.TException;
-import java.util.Map;
-
/**
- * This is here mostly for backwards compatibility.
- * Please see {@link org.apache.storm.LocalCluster} for
- * more details on testing a Storm Topology.
+ * This is here mostly for backwards compatibility. Please see {@link org.apache.storm.LocalCluster} for more details on testing a Storm
+ * Topology.
*/
public interface ILocalClusterTrackedTopologyAware extends ILocalCluster {
/**
* Submit a tracked topology to be run in local mode
+ *
* @param topologyName the name of the topology to use
- * @param conf the config for the topology
- * @param topology the topology itself.
+ * @param conf the config for the topology
+ * @param topology the topology itself.
* @return an AutoCloseable that will kill the topology.
+ *
* @throws TException on any error from nimbus
*/
ILocalTopology submitTopology(String topologyName, Map<String, Object> conf, TrackedTopology topology) throws TException;
/**
* Submit a tracked topology to be run in local mode
+ *
* @param topologyName the name of the topology to use
- * @param conf the config for the topology
- * @param topology the topology itself.
- * @param submitOpts options for topology
+ * @param conf the config for the topology
+ * @param topology the topology itself.
+ * @param submitOpts options for topology
* @return an AutoCloseable that will kill the topology.
+ *
* @throws TException on any error from nimbus
*/
- ILocalTopology submitTopologyWithOpts(String topologyName, Map<String, Object> conf, TrackedTopology topology, SubmitOptions submitOpts) throws TException;
+ ILocalTopology submitTopologyWithOpts(String topologyName, Map<String, Object> conf, TrackedTopology topology,
+ SubmitOptions submitOpts) throws TException;
}
[13/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SupervisorUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SupervisorUtils.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SupervisorUtils.java
index 43c125d..90d68dc 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SupervisorUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/SupervisorUtils.java
@@ -24,15 +24,14 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.generated.LSWorkerHeartbeat;
import org.apache.storm.localizer.LocalResource;
import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.ServerUtils;
import org.apache.storm.utils.Utils;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.LocalState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,9 +41,11 @@ public class SupervisorUtils {
private static final SupervisorUtils INSTANCE = new SupervisorUtils();
private static SupervisorUtils _instance = INSTANCE;
+
public static void setInstance(SupervisorUtils u) {
_instance = u;
}
+
public static void resetInstance() {
_instance = INSTANCE;
}
@@ -62,9 +63,9 @@ public class SupervisorUtils {
}
/**
- * Given the blob information returns the value of the uncompress field, handling it either being a string or a boolean value, or if it's not specified then
- * returns false
- *
+ * Given the blob information returns the value of the uncompress field, handling it either being a string or a boolean value, or if
+ * it's not specified then returns false
+ *
* @param blobInfo
* @return
*/
@@ -73,8 +74,8 @@ public class SupervisorUtils {
}
/**
- * Given the blob information returns the value of the workerRestart field, handling it either being a string or a boolean value, or
- * if it's not specified then returns false.
+ * Given the blob information returns the value of the workerRestart field, handling it either being a string or a boolean value, or if
+ * it's not specified then returns false.
*
* @param blobInfo the info for the blob.
* @return true if the blob needs a worker restart by way of the callback else false.
@@ -85,7 +86,7 @@ public class SupervisorUtils {
/**
* Returns a list of LocalResources based on the blobstore-map passed in
- *
+ *
* @param blobstoreMap
* @return
*/
@@ -94,7 +95,8 @@ public class SupervisorUtils {
if (blobstoreMap != null) {
for (Map.Entry<String, Map<String, Object>> map : blobstoreMap.entrySet()) {
Map<String, Object> blobConf = map.getValue();
- LocalResource localResource = new LocalResource(map.getKey(), shouldUncompressBlob(blobConf), blobNeedsWorkerRestart(blobConf));
+ LocalResource localResource =
+ new LocalResource(map.getKey(), shouldUncompressBlob(blobConf), blobNeedsWorkerRestart(blobConf));
localResourceList.add(localResource);
}
}
@@ -111,12 +113,30 @@ public class SupervisorUtils {
*
* @param conf
* @return
+ *
* @throws Exception
*/
public static Map<String, LSWorkerHeartbeat> readWorkerHeartbeats(Map<String, Object> conf) throws Exception {
return _instance.readWorkerHeartbeatsImpl(conf);
}
+ /**
+ * get worker heartbeat by workerId
+ *
+ * @param conf
+ * @param workerId
+ * @return
+ *
+ * @throws IOException
+ */
+ private static LSWorkerHeartbeat readWorkerHeartbeat(Map<String, Object> conf, String workerId) {
+ return _instance.readWorkerHeartbeatImpl(conf, workerId);
+ }
+
+ public static boolean isWorkerHbTimedOut(int now, LSWorkerHeartbeat whb, Map<String, Object> conf) {
+ return _instance.isWorkerHbTimedOutImpl(now, whb, conf);
+ }
+
public Map<String, LSWorkerHeartbeat> readWorkerHeartbeatsImpl(Map<String, Object> conf) throws Exception {
Map<String, LSWorkerHeartbeat> workerHeartbeats = new HashMap<>();
@@ -130,19 +150,6 @@ public class SupervisorUtils {
return workerHeartbeats;
}
-
- /**
- * get worker heartbeat by workerId
- *
- * @param conf
- * @param workerId
- * @return
- * @throws IOException
- */
- private static LSWorkerHeartbeat readWorkerHeartbeat(Map<String, Object> conf, String workerId) {
- return _instance.readWorkerHeartbeatImpl(conf, workerId);
- }
-
protected LSWorkerHeartbeat readWorkerHeartbeatImpl(Map<String, Object> conf, String workerId) {
try {
LocalState localState = ConfigUtils.workerState(conf, workerId);
@@ -153,11 +160,7 @@ public class SupervisorUtils {
}
}
- public static boolean isWorkerHbTimedOut(int now, LSWorkerHeartbeat whb, Map<String, Object> conf) {
- return _instance.isWorkerHbTimedOutImpl(now, whb, conf);
- }
-
- private boolean isWorkerHbTimedOutImpl(int now, LSWorkerHeartbeat whb, Map<String, Object> conf) {
+ private boolean isWorkerHbTimedOutImpl(int now, LSWorkerHeartbeat whb, Map<String, Object> conf) {
return (now - whb.get_time_secs()) > ObjectReader.getInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS));
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/UniFunc.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/UniFunc.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/UniFunc.java
index 9662af5..e531cb6 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/UniFunc.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/UniFunc.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.supervisor;
public interface UniFunc<T> {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeats.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeats.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeats.java
index c01819b..6e4814c 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeats.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/ReportWorkerHeartbeats.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.daemon.supervisor.timer;
@@ -21,7 +15,6 @@ package org.apache.storm.daemon.supervisor.timer;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.daemon.supervisor.Supervisor;
import org.apache.storm.daemon.supervisor.SupervisorUtils;
import org.apache.storm.generated.LSWorkerHeartbeat;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java
index 9ffce91..f68cb8c 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java
@@ -19,7 +19,6 @@
package org.apache.storm.daemon.supervisor.timer;
import java.util.Map;
-
import org.apache.storm.daemon.supervisor.Supervisor;
import org.apache.storm.healthcheck.HealthChecker;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
index 14ecf94..24d4471 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.daemon.supervisor.timer;
@@ -56,7 +50,7 @@ public class SupervisorHeartbeat implements Runnable {
List<Long> usedPorts = new ArrayList<>();
usedPorts.addAll(supervisor.getCurrAssignment().get().keySet());
supervisorInfo.set_used_ports(usedPorts);
- List metaDatas = (List)supervisor.getiSupervisor().getMetadata();
+ List metaDatas = (List) supervisor.getiSupervisor().getMetadata();
List<Long> portList = new ArrayList<>();
if (metaDatas != null) {
for (Object data : metaDatas) {
@@ -85,8 +79,8 @@ public class SupervisorHeartbeat implements Runnable {
// If configs are present in Generic map and legacy - the legacy values will be overwritten
- Map<String, Number> rawResourcesMap = (Map<String,Number>) conf.getOrDefault(
- Config.SUPERVISOR_RESOURCES_MAP, Collections.emptyMap()
+ Map<String, Number> rawResourcesMap = (Map<String, Number>) conf.getOrDefault(
+ Config.SUPERVISOR_RESOURCES_MAP, Collections.emptyMap()
);
for (Map.Entry<String, Number> stringNumberEntry : rawResourcesMap.entrySet()) {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SynchronizeAssignments.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SynchronizeAssignments.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SynchronizeAssignments.java
index ba8d133..639c9db 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SynchronizeAssignments.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SynchronizeAssignments.java
@@ -1,26 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.daemon.supervisor.timer;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.storm.cluster.IStormClusterState;
import org.apache.storm.daemon.supervisor.ReadClusterState;
import org.apache.storm.daemon.supervisor.Supervisor;
@@ -56,6 +49,18 @@ public class SynchronizeAssignments implements Runnable {
this.readClusterState = readClusterState;
}
+ private static void assignedAssignmentsToLocal(IStormClusterState clusterState, SupervisorAssignments assignments) {
+ if (null == assignments) {
+ //unknown error, just skip
+ return;
+ }
+ Map<String, byte[]> serAssignments = new HashMap<>();
+ for (Map.Entry<String, Assignment> entry : assignments.get_storm_assignment().entrySet()) {
+ serAssignments.put(entry.getKey(), Utils.serialize(entry.getValue()));
+ }
+ clusterState.syncRemoteAssignments(serAssignments);
+ }
+
@Override
public void run() {
// first sync assignments to local, then sync processes.
@@ -118,16 +123,4 @@ public class SynchronizeAssignments implements Runnable {
}
}
}
-
- private static void assignedAssignmentsToLocal(IStormClusterState clusterState, SupervisorAssignments assignments) {
- if (null == assignments) {
- //unknown error, just skip
- return;
- }
- Map<String, byte[]> serAssignments = new HashMap<>();
- for (Map.Entry<String, Assignment> entry : assignments.get_storm_assignment().entrySet()) {
- serAssignments.put(entry.getKey(), Utils.serialize(entry.getValue()));
- }
- clusterState.syncRemoteAssignments(serAssignments);
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/event/EventManager.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/event/EventManager.java b/storm-server/src/main/java/org/apache/storm/event/EventManager.java
index 64536c1..ff7342b 100644
--- a/storm-server/src/main/java/org/apache/storm/event/EventManager.java
+++ b/storm-server/src/main/java/org/apache/storm/event/EventManager.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.event;
public interface EventManager extends AutoCloseable {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/event/EventManagerImp.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/event/EventManagerImp.java b/storm-server/src/main/java/org/apache/storm/event/EventManagerImp.java
index 6b9d4f1..a527315 100644
--- a/storm-server/src/main/java/org/apache/storm/event/EventManagerImp.java
+++ b/storm-server/src/main/java/org/apache/storm/event/EventManagerImp.java
@@ -1,32 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.event;
-import org.apache.storm.utils.Utils;
-import org.apache.storm.utils.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+package org.apache.storm.event;
import java.io.InterruptedIOException;
import java.nio.channels.ClosedByInterruptException;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class EventManagerImp implements EventManager {
private static final Logger LOG = LoggerFactory.getLogger(EventManagerImp.class);
@@ -55,7 +49,7 @@ public class EventManagerImp implements EventManager {
r.run();
proccessInc();
} catch (Throwable t) {
- if (Utils.exceptionCauseIsInstanceOf(InterruptedIOException.class, t) ) {
+ if (Utils.exceptionCauseIsInstanceOf(InterruptedIOException.class, t)) {
LOG.info("Event manager interrupted while doing IO");
} else if (Utils.exceptionCauseIsInstanceOf(ClosedByInterruptException.class, t)) {
LOG.info("Event manager interrupted while doing NIO");
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/healthcheck/HealthChecker.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/healthcheck/HealthChecker.java b/storm-server/src/main/java/org/apache/storm/healthcheck/HealthChecker.java
index c373b05..83faf13 100644
--- a/storm-server/src/main/java/org/apache/storm/healthcheck/HealthChecker.java
+++ b/storm-server/src/main/java/org/apache/storm/healthcheck/HealthChecker.java
@@ -18,12 +18,6 @@
package org.apache.storm.healthcheck;
-import org.apache.storm.DaemonConfig;
-import org.apache.storm.utils.ObjectReader;
-import org.apache.storm.utils.ServerConfigUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStream;
@@ -32,6 +26,11 @@ import java.nio.channels.ClosedByInterruptException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.storm.DaemonConfig;
+import org.apache.storm.utils.ObjectReader;
+import org.apache.storm.utils.ServerConfigUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HealthChecker {
@@ -50,8 +49,9 @@ public class HealthChecker {
if (parentFile.exists()) {
File[] list = parentFile.listFiles();
for (File f : list) {
- if (!f.isDirectory() && f.canExecute())
+ if (!f.isDirectory() && f.canExecute()) {
healthScripts.add(f.getAbsolutePath());
+ }
}
}
for (String script : healthScripts) {
@@ -66,7 +66,7 @@ public class HealthChecker {
// to execute properly, not that the system is unhealthy, in which case
// we don't want to start killing things.
- if (results.contains(FAILED)|| results.contains(FAILED_WITH_EXIT_CODE)
+ if (results.contains(FAILED) || results.contains(FAILED_WITH_EXIT_CODE)
|| results.contains(TIMEOUT)) {
LOG.warn("The supervisor healthchecks failed!!!");
return 1;
@@ -120,8 +120,9 @@ public class HealthChecker {
LOG.warn("Script failed with exception: ", e);
return FAILED_WITH_EXIT_CODE;
} finally {
- if (interruptThread != null)
+ if (interruptThread != null) {
interruptThread.interrupt();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/localizer/AsyncLocalizer.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/AsyncLocalizer.java b/storm-server/src/main/java/org/apache/storm/localizer/AsyncLocalizer.java
index 1cdcaad..8b41c68 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/AsyncLocalizer.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/AsyncLocalizer.java
@@ -29,7 +29,6 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -68,18 +67,19 @@ public class AsyncLocalizer implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(AsyncLocalizer.class);
private static final CompletableFuture<Void> ALL_DONE_FUTURE = new CompletableFuture<>();
+ private static final int ATTEMPTS_INTERVAL_TIME = 100;
static {
ALL_DONE_FUTURE.complete(null);
}
- private final boolean isLocalMode;
// track resources - user to resourceSet
//ConcurrentHashMap is explicitly used everywhere in this class because it uses locks to guarantee atomicity for compute and
// computeIfAbsent where as ConcurrentMap allows for a retry of the function passed in, and would require the function to have
// no side effects.
protected final ConcurrentHashMap<String, ConcurrentHashMap<String, LocalizedResource>> userFiles = new ConcurrentHashMap<>();
protected final ConcurrentHashMap<String, ConcurrentHashMap<String, LocalizedResource>> userArchives = new ConcurrentHashMap<>();
+ private final boolean isLocalMode;
// topology to tracking of topology dir and resources
private final ConcurrentHashMap<String, CompletableFuture<Void>> blobPending;
private final Map<String, Object> conf;
@@ -87,16 +87,14 @@ public class AsyncLocalizer implements AutoCloseable {
private final boolean symlinksDisabled;
private final ConcurrentHashMap<String, LocallyCachedBlob> topologyBlobs = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, CompletableFuture<Void>> topologyBasicDownloaded = new ConcurrentHashMap<>();
-
private final Path localBaseDir;
private final int blobDownloadRetries;
private final ScheduledExecutorService execService;
-
+ private final long cacheCleanupPeriod;
// cleanup
@VisibleForTesting
protected long cacheTargetSize;
- private final long cacheCleanupPeriod;
-
+
@VisibleForTesting
AsyncLocalizer(Map<String, Object> conf, AdvancedFSOps ops, String baseDir) throws IOException {
@@ -106,7 +104,7 @@ public class AsyncLocalizer implements AutoCloseable {
localBaseDir = Paths.get(baseDir);
// default cache size 10GB, converted to Bytes
cacheTargetSize = ObjectReader.getInt(conf.get(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_TARGET_SIZE_MB),
- 10 * 1024).longValue() << 20;
+ 10 * 1024).longValue() << 20;
// default 30 seconds. (we cache the size so it is cheap to do)
cacheCleanupPeriod = ObjectReader.getInt(conf.get(
DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS), 30 * 1000).longValue();
@@ -117,10 +115,10 @@ public class AsyncLocalizer implements AutoCloseable {
DaemonConfig.SUPERVISOR_BLOBSTORE_DOWNLOAD_MAX_RETRIES), 3);
execService = Executors.newScheduledThreadPool(threadPoolSize,
- new ThreadFactoryBuilder().setNameFormat("AsyncLocalizer Executor - %d").build());
+ new ThreadFactoryBuilder().setNameFormat("AsyncLocalizer Executor - %d").build());
reconstructLocalizedResources();
- symlinksDisabled = (boolean)conf.getOrDefault(Config.DISABLE_SYMLINKS, false);
+ symlinksDisabled = (boolean) conf.getOrDefault(Config.DISABLE_SYMLINKS, false);
blobPending = new ConcurrentHashMap<>();
}
@@ -131,40 +129,43 @@ public class AsyncLocalizer implements AutoCloseable {
@VisibleForTesting
LocallyCachedBlob getTopoJar(final String topologyId) {
return topologyBlobs.computeIfAbsent(ConfigUtils.masterStormJarKey(topologyId),
- (tjk) -> {
- try {
- return new LocallyCachedTopologyBlob(topologyId, isLocalMode, conf, fsOps,
- LocallyCachedTopologyBlob.TopologyBlobType.TOPO_JAR);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
+ (tjk) -> {
+ try {
+ return new LocallyCachedTopologyBlob(topologyId, isLocalMode, conf, fsOps,
+ LocallyCachedTopologyBlob.TopologyBlobType
+ .TOPO_JAR);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
@VisibleForTesting
LocallyCachedBlob getTopoCode(final String topologyId) {
return topologyBlobs.computeIfAbsent(ConfigUtils.masterStormCodeKey(topologyId),
- (tck) -> {
- try {
- return new LocallyCachedTopologyBlob(topologyId, isLocalMode, conf, fsOps,
- LocallyCachedTopologyBlob.TopologyBlobType.TOPO_CODE);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
+ (tck) -> {
+ try {
+ return new LocallyCachedTopologyBlob(topologyId, isLocalMode, conf, fsOps,
+ LocallyCachedTopologyBlob.TopologyBlobType
+ .TOPO_CODE);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
@VisibleForTesting
LocallyCachedBlob getTopoConf(final String topologyId) {
return topologyBlobs.computeIfAbsent(ConfigUtils.masterStormConfKey(topologyId),
- (tck) -> {
- try {
- return new LocallyCachedTopologyBlob(topologyId, isLocalMode, conf, fsOps,
- LocallyCachedTopologyBlob.TopologyBlobType.TOPO_CONF);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
+ (tck) -> {
+ try {
+ return new LocallyCachedTopologyBlob(topologyId, isLocalMode, conf, fsOps,
+ LocallyCachedTopologyBlob.TopologyBlobType
+ .TOPO_CONF);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
}
private LocalizedResource getUserArchive(String user, String key) {
@@ -181,33 +182,35 @@ public class AsyncLocalizer implements AutoCloseable {
/**
* Request that all of the blobs necessary for this topology be downloaded.
+ *
* @param assignment the assignment that needs the blobs
- * @param port the port the assignment is a part of
- * @param cb a callback for when the blobs change. This is only for blobs that are tied to the lifetime of the worker.
+ * @param port the port the assignment is a part of
+ * @param cb a callback for when the blobs change. This is only for blobs that are tied to the lifetime of the worker.
* @return a Future that indicates when they are all downloaded.
+ *
* @throws IOException if there was an error while trying doing it.
*/
public CompletableFuture<Void> requestDownloadTopologyBlobs(final LocalAssignment assignment, final int port,
- final BlobChangingCallback cb) throws IOException {
+ final BlobChangingCallback cb) throws IOException {
final PortAndAssignment pna = new PortAndAssignment(port, assignment);
final String topologyId = pna.getToplogyId();
CompletableFuture<Void> baseBlobs = requestDownloadBaseTopologyBlobs(pna, cb);
return baseBlobs.thenComposeAsync((v) ->
- blobPending.compute(topologyId, (tid, old) -> {
- CompletableFuture<Void> ret = old;
- if (ret == null) {
- ret = CompletableFuture.supplyAsync(new DownloadBlobs(pna, cb), execService);
- } else {
- try {
- addReferencesToBlobs(pna, cb);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- LOG.debug("Reserved blobs {} {}", topologyId, ret);
- return ret;
- }));
+ blobPending.compute(topologyId, (tid, old) -> {
+ CompletableFuture<Void> ret = old;
+ if (ret == null) {
+ ret = CompletableFuture.supplyAsync(new DownloadBlobs(pna, cb), execService);
+ } else {
+ try {
+ addReferencesToBlobs(pna, cb);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ LOG.debug("Reserved blobs {} {}", topologyId, ret);
+ return ret;
+ }));
}
@VisibleForTesting
@@ -225,19 +228,17 @@ public class AsyncLocalizer implements AutoCloseable {
topoConf.addReference(pna, cb);
return topologyBasicDownloaded.computeIfAbsent(topologyId,
- (tid) -> downloadOrUpdate(topoJar, topoCode, topoConf));
+ (tid) -> downloadOrUpdate(topoJar, topoCode, topoConf));
}
- private static final int ATTEMPTS_INTERVAL_TIME = 100;
-
- private CompletableFuture<Void> downloadOrUpdate(LocallyCachedBlob ... blobs) {
+ private CompletableFuture<Void> downloadOrUpdate(LocallyCachedBlob... blobs) {
return downloadOrUpdate(Arrays.asList(blobs));
}
private CompletableFuture<Void> downloadOrUpdate(Collection<? extends LocallyCachedBlob> blobs) {
- CompletableFuture<Void> [] all = new CompletableFuture[blobs.size()];
+ CompletableFuture<Void>[] all = new CompletableFuture[blobs.size()];
int i = 0;
- for (final LocallyCachedBlob blob: blobs) {
+ for (final LocallyCachedBlob blob : blobs) {
all[i] = CompletableFuture.runAsync(() -> {
LOG.debug("STARTING download of {}", blob);
try (ClientBlobStore blobStore = getClientBlobStore()) {
@@ -296,7 +297,7 @@ public class AsyncLocalizer implements AutoCloseable {
futures.add(downloadOrUpdate(map.values()));
}
}
- for (CompletableFuture<?> f: futures) {
+ for (CompletableFuture<?> f : futures) {
try {
f.get();
} catch (Exception e) {
@@ -312,8 +313,7 @@ public class AsyncLocalizer implements AutoCloseable {
}
/**
- * Start any background threads needed. This includes updating blobs and cleaning up
- * unused blobs over the configured size limit.
+ * Start any background threads needed. This includes updating blobs and cleaning up unused blobs over the configured size limit.
*/
public void start() {
execService.scheduleWithFixedDelay(this::updateBlobs, 30, 30, TimeUnit.SECONDS);
@@ -366,76 +366,17 @@ public class AsyncLocalizer implements AutoCloseable {
}
}
- private class DownloadBlobs implements Supplier<Void> {
- private final PortAndAssignment pna;
- private final BlobChangingCallback cb;
-
- public DownloadBlobs(PortAndAssignment pna, BlobChangingCallback cb) {
- this.pna = pna;
- this.cb = cb;
- }
-
- @Override
- public Void get() {
- try {
- String topologyId = pna.getToplogyId();
- String topoOwner = pna.getOwner();
- String stormroot = ConfigUtils.supervisorStormDistRoot(conf, topologyId);
- Map<String, Object> topoConf = ConfigUtils.readSupervisorStormConf(conf, topologyId);
-
- @SuppressWarnings("unchecked")
- Map<String, Map<String, Object>> blobstoreMap =
- (Map<String, Map<String, Object>>) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
-
- List<LocalResource> localResourceList = getLocalResources(pna);
- if (!localResourceList.isEmpty()) {
- File userDir = getLocalUserFileCacheDir(topoOwner);
- if (!fsOps.fileExists(userDir)) {
- fsOps.forceMkdir(userDir);
- }
- List<LocalizedResource> localizedResources = getBlobs(localResourceList, pna, cb);
- fsOps.setupBlobPermissions(userDir, topoOwner);
- if (!symlinksDisabled) {
- for (LocalizedResource localizedResource : localizedResources) {
- String keyName = localizedResource.getKey();
- //The sym link we are pointing to
- File rsrcFilePath = localizedResource.getCurrentSymlinkPath().toFile();
-
- String symlinkName = null;
- if (blobstoreMap != null) {
- Map<String, Object> blobInfo = blobstoreMap.get(keyName);
- if (blobInfo != null && blobInfo.containsKey("localname")) {
- symlinkName = (String) blobInfo.get("localname");
- } else {
- symlinkName = keyName;
- }
- } else {
- // all things are from dependencies
- symlinkName = keyName;
- }
- fsOps.createSymlink(new File(stormroot, symlinkName), rsrcFilePath);
- }
- }
- }
-
- return null;
- } catch (Exception e) {
- LOG.warn("Caught Exception While Downloading (rethrowing)... ", e);
- throw new RuntimeException(e);
- }
- }
- }
-
/**
* Do everything needed to recover the state in the AsyncLocalizer for a running topology.
+ *
* @param currentAssignment the assignment for the topology.
- * @param port the port the assignment is on.
- * @param cb a callback for when the blobs are updated. This will only be for blobs that
- * indicate that if they change the worker should be restarted.
+ * @param port the port the assignment is on.
+ * @param cb a callback for when the blobs are updated. This will only be for blobs that indicate that if they change
+ * the worker should be restarted.
* @throws IOException on any error trying to recover the state.
*/
public void recoverRunningTopology(final LocalAssignment currentAssignment, final int port,
- final BlobChangingCallback cb) throws IOException {
+ final BlobChangingCallback cb) throws IOException {
final PortAndAssignment pna = new PortAndAssignment(port, currentAssignment);
final String topologyId = pna.getToplogyId();
@@ -463,7 +404,7 @@ public class AsyncLocalizer implements AutoCloseable {
* Remove this assignment/port as blocking resources from being cleaned up.
*
* @param assignment the assignment the resources are for
- * @param port the port the topology is running on
+ * @param port the port the topology is running on
* @throws IOException on any error
*/
public void releaseSlotFor(LocalAssignment assignment, int port) throws IOException {
@@ -512,13 +453,13 @@ public class AsyncLocalizer implements AutoCloseable {
}
private void recoverLocalizedArchivesForUser(String user) throws IOException {
- for (String key: LocalizedResource.getLocalizedArchiveKeys(localBaseDir, user)) {
+ for (String key : LocalizedResource.getLocalizedArchiveKeys(localBaseDir, user)) {
getUserArchive(user, key);
}
}
private void recoverLocalizedFilesForUser(String user) throws IOException {
- for (String key: LocalizedResource.getLocalizedFileKeys(localBaseDir, user)) {
+ for (String key : LocalizedResource.getLocalizedFileKeys(localBaseDir, user)) {
getUserFile(user, key);
}
}
@@ -544,7 +485,7 @@ public class AsyncLocalizer implements AutoCloseable {
// ignores invalid user/topo/key
void removeBlobReference(String key, PortAndAssignment pna,
- boolean uncompress) throws AuthorizationException, KeyNotFoundException {
+ boolean uncompress) throws AuthorizationException, KeyNotFoundException {
String user = pna.getOwner();
String topo = pna.getToplogyId();
ConcurrentMap<String, LocalizedResource> lrsrcSet = uncompress ? userArchives.get(user) : userFiles.get(user);
@@ -555,11 +496,11 @@ public class AsyncLocalizer implements AutoCloseable {
lrsrc.removeReference(pna);
} else {
LOG.warn("trying to remove non-existent blob, key: " + key + " for user: " + user
- + " topo: " + topo);
+ + " topo: " + topo);
}
} else {
LOG.warn("trying to remove blob for non-existent resource set for user: " + user + " key: "
- + key + " topo: " + topo);
+ + key + " topo: " + topo);
}
}
@@ -568,9 +509,8 @@ public class AsyncLocalizer implements AutoCloseable {
}
/**
- * This function either returns the blobs in the existing cache or if they don't exist in the
- * cache, it downloads them in parallel (up to SUPERVISOR_BLOBSTORE_DOWNLOAD_THREAD_COUNT)
- * and will block until all of them have been downloaded.
+ * This function either returns the blobs in the existing cache or if they don't exist in the cache, it downloads them in parallel (up
+ * to SUPERVISOR_BLOBSTORE_DOWNLOAD_THREAD_COUNT) and will block until all of them have been downloaded.
*/
List<LocalizedResource> getBlobs(List<LocalResource> localResources, PortAndAssignment pna, BlobChangingCallback cb)
throws AuthorizationException, KeyNotFoundException, IOException {
@@ -582,7 +522,7 @@ public class AsyncLocalizer implements AutoCloseable {
List<CompletableFuture<?>> futures = new ArrayList<>();
try {
- for (LocalResource localResource: localResources) {
+ for (LocalResource localResource : localResources) {
String key = localResource.getBlobName();
boolean uncompress = localResource.shouldUncompress();
LocalizedResource lrsrc = uncompress ? getUserArchive(user, key) : getUserFile(user, key);
@@ -595,7 +535,7 @@ public class AsyncLocalizer implements AutoCloseable {
lrsrc.addReference(pna, localResource.needsCallback() ? cb : null);
}
- for (CompletableFuture<?> futureRsrc: futures) {
+ for (CompletableFuture<?> futureRsrc : futures) {
futureRsrc.get();
}
} catch (ExecutionException e) {
@@ -610,10 +550,6 @@ public class AsyncLocalizer implements AutoCloseable {
return results;
}
- private interface ConsumePathAndId {
- void accept(Path path, String topologyId) throws IOException;
- }
-
private void forEachTopologyDistDir(ConsumePathAndId consumer) throws IOException {
Path stormCodeRoot = Paths.get(ConfigUtils.supervisorStormDistRoot(conf));
if (Files.exists(stormCodeRoot) && Files.isDirectory(stormCodeRoot)) {
@@ -669,7 +605,7 @@ public class AsyncLocalizer implements AutoCloseable {
LOG.debug("Resource cleanup: {}", toClean);
Set<String> allUsers = new HashSet<>(userArchives.keySet());
allUsers.addAll(userFiles.keySet());
- for (String user: allUsers) {
+ for (String user : allUsers) {
ConcurrentMap<String, LocalizedResource> filesForUser = userFiles.get(user);
ConcurrentMap<String, LocalizedResource> archivesForUser = userArchives.get(user);
if ((filesForUser == null || filesForUser.size() == 0)
@@ -686,4 +622,68 @@ public class AsyncLocalizer implements AutoCloseable {
}
}
}
+
+ private interface ConsumePathAndId {
+ void accept(Path path, String topologyId) throws IOException;
+ }
+
+ private class DownloadBlobs implements Supplier<Void> {
+ private final PortAndAssignment pna;
+ private final BlobChangingCallback cb;
+
+ public DownloadBlobs(PortAndAssignment pna, BlobChangingCallback cb) {
+ this.pna = pna;
+ this.cb = cb;
+ }
+
+ @Override
+ public Void get() {
+ try {
+ String topologyId = pna.getToplogyId();
+ String topoOwner = pna.getOwner();
+ String stormroot = ConfigUtils.supervisorStormDistRoot(conf, topologyId);
+ Map<String, Object> topoConf = ConfigUtils.readSupervisorStormConf(conf, topologyId);
+
+ @SuppressWarnings("unchecked")
+ Map<String, Map<String, Object>> blobstoreMap =
+ (Map<String, Map<String, Object>>) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
+
+ List<LocalResource> localResourceList = getLocalResources(pna);
+ if (!localResourceList.isEmpty()) {
+ File userDir = getLocalUserFileCacheDir(topoOwner);
+ if (!fsOps.fileExists(userDir)) {
+ fsOps.forceMkdir(userDir);
+ }
+ List<LocalizedResource> localizedResources = getBlobs(localResourceList, pna, cb);
+ fsOps.setupBlobPermissions(userDir, topoOwner);
+ if (!symlinksDisabled) {
+ for (LocalizedResource localizedResource : localizedResources) {
+ String keyName = localizedResource.getKey();
+ //The sym link we are pointing to
+ File rsrcFilePath = localizedResource.getCurrentSymlinkPath().toFile();
+
+ String symlinkName = null;
+ if (blobstoreMap != null) {
+ Map<String, Object> blobInfo = blobstoreMap.get(keyName);
+ if (blobInfo != null && blobInfo.containsKey("localname")) {
+ symlinkName = (String) blobInfo.get("localname");
+ } else {
+ symlinkName = keyName;
+ }
+ } else {
+ // all things are from dependencies
+ symlinkName = keyName;
+ }
+ fsOps.createSymlink(new File(stormroot, symlinkName), rsrcFilePath);
+ }
+ }
+ }
+
+ return null;
+ } catch (Exception e) {
+ LOG.warn("Caught Exception While Downloading (rethrowing)... ", e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/localizer/BlobChangingCallback.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/BlobChangingCallback.java b/storm-server/src/main/java/org/apache/storm/localizer/BlobChangingCallback.java
index e99f95b..ebbbdc5 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/BlobChangingCallback.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/BlobChangingCallback.java
@@ -26,15 +26,15 @@ import org.apache.storm.generated.LocalAssignment;
public interface BlobChangingCallback {
/**
- * Informs the listener that a blob has changed and is ready to update and replace a localized blob that has been marked as
- * tied to the life cycle of the worker process.
+ * Informs the listener that a blob has changed and is ready to update and replace a localized blob that has been marked as tied to the
+ * life cycle of the worker process.
+ *
+ * If `go.getLatch()` is never called before the method completes it is assumed that the listener is good with the blob changing.
*
- * If `go.getLatch()` is never called before the method completes it is assumed that
- * the listener is good with the blob changing.
* @param assignment the assignment this resource and callback are registered with.
- * @param port the port that this resource and callback are registered with.
- * @param blob the blob that is going to change.
- * @param go a way to indicate if the listener is ready for the resource to change.
+ * @param port the port that this resource and callback are registered with.
+ * @param blob the blob that is going to change.
+ * @param go a way to indicate if the listener is ready for the resource to change.
*/
void blobChanging(LocalAssignment assignment, int port, LocallyCachedBlob blob, GoodToGo go);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/localizer/GoodToGo.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/GoodToGo.java b/storm-server/src/main/java/org/apache/storm/localizer/GoodToGo.java
index 2fffe21..9217388 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/GoodToGo.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/GoodToGo.java
@@ -22,39 +22,20 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Future;
/**
- * Used as a way to give feedback that the listener is ready for the caller to change the blob.
- * By calling @{link GoodToGo#getLatch()} the listener indicates that it wants to block
- * changing the blob until the CountDownLatch is triggered with a call to @{link CountDownLatch#countDown()}.
+ * Used as a way to give feedback that the listener is ready for the caller to change the blob. By calling @{link GoodToGo#getLatch()} the
+ * listener indicates that it wants to block changing the blob until the CountDownLatch is triggered with a call to @{link
+ * CountDownLatch#countDown()}.
*/
public class GoodToGo {
- public static class GoodToGoLatch {
- private final CountDownLatch latch;
- private final Future<Void> doneChanging;
- private boolean wasCounted = false;
-
- public GoodToGoLatch(CountDownLatch latch, Future<Void> doneChanging) {
- this.latch = latch;
- this.doneChanging = doneChanging;
- }
-
- public synchronized Future<Void> countDown() {
- if (!wasCounted) {
- latch.countDown();
- wasCounted = true;
- }
- return doneChanging;
- }
- }
-
private final GoodToGoLatch latch;
private boolean gotLatch = false;
-
public GoodToGo(CountDownLatch latch, Future<Void> doneChanging) {
this.latch = new GoodToGoLatch(latch, doneChanging);
}
/**
* Get the latch and indicate that you want to block the blob being changed.
+ *
* @return the latch to use when you are ready.
*/
public synchronized GoodToGoLatch getLatch() {
@@ -67,4 +48,23 @@ public class GoodToGo {
latch.countDown();
}
}
+
+ public static class GoodToGoLatch {
+ private final CountDownLatch latch;
+ private final Future<Void> doneChanging;
+ private boolean wasCounted = false;
+
+ public GoodToGoLatch(CountDownLatch latch, Future<Void> doneChanging) {
+ this.latch = latch;
+ this.doneChanging = doneChanging;
+ }
+
+ public synchronized Future<Void> countDown() {
+ if (!wasCounted) {
+ latch.countDown();
+ wasCounted = true;
+ }
+ return doneChanging;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/localizer/LocalResource.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalResource.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalResource.java
index 862349b..f5dfc77 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocalResource.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalResource.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.localizer;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
index 75a2d6d..6e0061d 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
@@ -18,8 +18,6 @@
package org.apache.storm.localizer;
-import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
-
import com.google.common.annotations.VisibleForTesting;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -57,13 +55,13 @@ import org.apache.storm.utils.ShellUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
+
/**
- * Represents a resource that is localized on the supervisor.
- * A localized resource has a .current symlink to the current version file which is named
- * filename.{current version}. There is also a filename.version which contains the latest version.
+ * Represents a resource that is localized on the supervisor. A localized resource has a .current symlink to the current version file which
+ * is named filename.{current version}. There is also a filename.version which contains the latest version.
*/
public class LocalizedResource extends LocallyCachedBlob {
- private static final Logger LOG = LoggerFactory.getLogger(LocalizedResource.class);
@VisibleForTesting
static final String CURRENT_BLOB_SUFFIX = ".current";
@VisibleForTesting
@@ -77,7 +75,34 @@ public class LocalizedResource extends LocallyCachedBlob {
static final String FILESDIR = "files";
@VisibleForTesting
static final String ARCHIVESDIR = "archives";
+ private static final Logger LOG = LoggerFactory.getLogger(LocalizedResource.class);
private static final String TO_UNCOMPRESS = "_tmp_";
+ private static final Pattern VERSION_FILE_PATTERN = Pattern.compile("^(.+)\\.(\\d+)$");
+ // filesystem path to the resource
+ private final Path baseDir;
+ private final Path versionFilePath;
+ private final Path symlinkPath;
+ private final boolean uncompressed;
+ private final IAdvancedFSOps fsOps;
+ private final String user;
+ private final Map<String, Object> conf;
+ // size of the resource
+ private long size = -1;
+
+ LocalizedResource(String key, Path localBaseDir, boolean uncompressed, IAdvancedFSOps fsOps, Map<String, Object> conf,
+ String user) {
+ super(key + (uncompressed ? " archive" : " file"), key);
+ Path base = getLocalUserFileCacheDir(localBaseDir, user);
+ this.baseDir = uncompressed ? getCacheDirForArchives(base) : getCacheDirForFiles(base);
+ this.conf = conf;
+ this.user = user;
+ this.fsOps = fsOps;
+ versionFilePath = constructVersionFileName(baseDir, key);
+ symlinkPath = constructBlobCurrentSymlinkName(baseDir, key);
+ this.uncompressed = uncompressed;
+ //Set the size in case we are recovering an already downloaded object
+ setSize();
+ }
private static Path constructVersionFileName(Path baseDir, String key) {
return baseDir.resolve(key + BLOB_VERSION_SUFFIX);
@@ -141,16 +166,16 @@ public class LocalizedResource extends LocallyCachedBlob {
return Collections.emptyList();
}
return Files.list(dir)
- .map((p) -> p.getFileName().toString())
- .filter((name) -> name.toLowerCase().endsWith(CURRENT_BLOB_SUFFIX))
- .map((key) -> {
- int p = key.lastIndexOf('.');
- if (p > 0) {
- key = key.substring(0, p);
- }
- return key;
- })
- .collect(Collectors.toList());
+ .map((p) -> p.getFileName().toString())
+ .filter((name) -> name.toLowerCase().endsWith(CURRENT_BLOB_SUFFIX))
+ .map((key) -> {
+ int p = key.lastIndexOf('.');
+ if (p > 0) {
+ key = key.substring(0, p);
+ }
+ return key;
+ })
+ .collect(Collectors.toList());
}
// baseDir/supervisor/usercache/
@@ -179,32 +204,6 @@ public class LocalizedResource extends LocallyCachedBlob {
return dir.resolve(ARCHIVESDIR);
}
- // filesystem path to the resource
- private final Path baseDir;
- private final Path versionFilePath;
- private final Path symlinkPath;
- private final boolean uncompressed;
- private final IAdvancedFSOps fsOps;
- private final String user;
- // size of the resource
- private long size = -1;
- private final Map<String, Object> conf;
-
- LocalizedResource(String key, Path localBaseDir, boolean uncompressed, IAdvancedFSOps fsOps, Map<String, Object> conf,
- String user) {
- super(key + (uncompressed ? " archive" : " file"), key);
- Path base = getLocalUserFileCacheDir(localBaseDir, user);
- this.baseDir = uncompressed ? getCacheDirForArchives(base) : getCacheDirForFiles(base);
- this.conf = conf;
- this.user = user;
- this.fsOps = fsOps;
- versionFilePath = constructVersionFileName(baseDir, key);
- symlinkPath = constructBlobCurrentSymlinkName(baseDir, key);
- this.uncompressed = uncompressed;
- //Set the size in case we are recovering an already downloaded object
- setSize();
- }
-
Path getCurrentSymlinkPath() {
return symlinkPath;
}
@@ -331,7 +330,7 @@ public class LocalizedResource extends LocallyCachedBlob {
LOG.warn("Exit code from worker-launcher is: {}", exitCode, e);
LOG.debug("output: {}", shExec.getOutput());
throw new IOException("Setting blob permissions failed"
- + " (exitCode=" + exitCode + ") with output: " + shExec.getOutput(), e);
+ + " (exitCode=" + exitCode + ") with output: " + shExec.getOutput(), e);
}
}
@@ -343,8 +342,6 @@ public class LocalizedResource extends LocallyCachedBlob {
return baseDir.resolve(Paths.get(LocalizedResource.TO_UNCOMPRESS + getKey() + CURRENT_BLOB_SUFFIX));
}
- private static final Pattern VERSION_FILE_PATTERN = Pattern.compile("^(.+)\\.(\\d+)$");
-
@Override
public void cleanupOrphanedData() throws IOException {
//There are a few possible files that we would want to clean up
@@ -434,14 +431,14 @@ public class LocalizedResource extends LocallyCachedBlob {
@Override
public boolean isFullyDownloaded() {
return Files.exists(getFilePathWithVersion())
- && Files.exists(getCurrentSymlinkPath())
- && Files.exists(versionFilePath);
+ && Files.exists(getCurrentSymlinkPath())
+ && Files.exists(versionFilePath);
}
@Override
public boolean equals(Object other) {
if (other instanceof LocalizedResource) {
- LocalizedResource l = (LocalizedResource)other;
+ LocalizedResource l = (LocalizedResource) other;
return getKey().equals(l.getKey()) && uncompressed == l.uncompressed && baseDir.equals(l.baseDir);
}
return false;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
index 936dbc1..15eba02 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.localizer;
@@ -37,11 +31,11 @@ import org.slf4j.LoggerFactory;
*/
public class LocalizedResourceRetentionSet {
public static final Logger LOG = LoggerFactory.getLogger(LocalizedResourceRetentionSet.class);
+ @VisibleForTesting
+ final SortedMap<LocallyCachedBlob, Map<String, ? extends LocallyCachedBlob>> noReferences;
private long currentSize;
// targetSize in Bytes
private long targetSize;
- @VisibleForTesting
- final SortedMap<LocallyCachedBlob, Map<String, ? extends LocallyCachedBlob>> noReferences;
LocalizedResourceRetentionSet(long targetSize) {
this(targetSize, new LRUComparator());
@@ -68,7 +62,7 @@ public class LocalizedResourceRetentionSet {
* if they are deleted on disk too.
*/
public void addResources(ConcurrentMap<String, ? extends LocallyCachedBlob> blobs) {
- for (LocallyCachedBlob b: blobs.values()) {
+ for (LocallyCachedBlob b : blobs.values()) {
currentSize += b.getSizeOnDisk();
if (b.isUsed()) {
LOG.debug("NOT going to clean up {}, {} depends on it", b.getKey(), b.getDependencies());
@@ -90,7 +84,7 @@ public class LocalizedResourceRetentionSet {
long bytesOver = currentSize - targetSize;
//First delete everything that no longer exists...
for (Iterator<Map.Entry<LocallyCachedBlob, Map<String, ? extends LocallyCachedBlob>>> i = noReferences.entrySet().iterator();
- i.hasNext();) {
+ i.hasNext(); ) {
Map.Entry<LocallyCachedBlob, Map<String, ? extends LocallyCachedBlob>> rsrc = i.next();
LocallyCachedBlob resource = rsrc.getKey();
try {
@@ -109,7 +103,7 @@ public class LocalizedResourceRetentionSet {
}
for (Iterator<Map.Entry<LocallyCachedBlob, Map<String, ? extends LocallyCachedBlob>>> i = noReferences.entrySet().iterator();
- bytesOver > 0 && i.hasNext();) {
+ bytesOver > 0 && i.hasNext(); ) {
Map.Entry<LocallyCachedBlob, Map<String, ? extends LocallyCachedBlob>> rsrc = i.next();
LocallyCachedBlob resource = rsrc.getKey();
Map<String, ? extends LocallyCachedBlob> set = rsrc.getValue();
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java
index 1cbb221..d112ee5 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.localizer;
@@ -43,16 +37,15 @@ import org.slf4j.LoggerFactory;
* Represents a blob that is cached locally on disk by the supervisor.
*/
public abstract class LocallyCachedBlob {
- private static final Logger LOG = LoggerFactory.getLogger(LocallyCachedBlob.class);
public static final long NOT_DOWNLOADED_VERSION = -1;
+ private static final Logger LOG = LoggerFactory.getLogger(LocallyCachedBlob.class);
// A callback that does nothing.
private static final BlobChangingCallback NOOP_CB = (assignment, port, resource, go) -> {
};
-
- private long lastUsed = Time.currentTimeMillis();
private final Map<PortAndAssignment, BlobChangingCallback> references = new HashMap<>();
private final String blobDescription;
private final String blobKey;
+ private long lastUsed = Time.currentTimeMillis();
private CompletableFuture<Void> doneUpdating = null;
/**
@@ -66,28 +59,8 @@ public abstract class LocallyCachedBlob {
this.blobKey = blobKey;
}
- /**
- * Get the version of the blob cached locally. If the version is unknown or it has not been downloaded NOT_DOWNLOADED_VERSION should be
- * returned. PRECONDITION: this can only be called with a lock on this instance held.
- */
- public abstract long getLocalVersion();
-
- /**
- * Get the version of the blob in the blob store. PRECONDITION: this can only be called with a lock on this instance held.
- */
- public abstract long getRemoteVersion(ClientBlobStore store) throws KeyNotFoundException, AuthorizationException;
-
- /**
- * Download the latest version to a temp location. This may also include unzipping some or all of the data to a temp location.
- * PRECONDITION: this can only be called with a lock on this instance held.
- *
- * @param store the store to us to download the data.
- * @return the version that was downloaded.
- */
- public abstract long downloadToTempLocation(ClientBlobStore store) throws IOException, KeyNotFoundException, AuthorizationException;
-
protected static long downloadToTempLocation(ClientBlobStore store, String key, long currentVersion, IAdvancedFSOps fsOps,
- Function<Long, Path> getTempPath)
+ Function<Long, Path> getTempPath)
throws KeyNotFoundException, AuthorizationException, IOException {
try (InputStreamWithMeta in = store.getBlob(key)) {
long newVersion = in.getVersion();
@@ -116,6 +89,51 @@ public abstract class LocallyCachedBlob {
}
/**
+ * Get the size of p in bytes.
+ * @param p the path to read.
+ * @return the size of p in bytes.
+ */
+ protected static long getSizeOnDisk(Path p) throws IOException {
+ if (!Files.exists(p)) {
+ return 0;
+ } else if (Files.isRegularFile(p)) {
+ return Files.size(p);
+ } else {
+ //We will not follow sym links
+ return Files.walk(p)
+ .filter((subp) -> Files.isRegularFile(subp, LinkOption.NOFOLLOW_LINKS))
+ .mapToLong((subp) -> {
+ try {
+ return Files.size(subp);
+ } catch (IOException e) {
+ LOG.warn("Could not get the size of ");
+ }
+ return 0;
+ }).sum();
+ }
+ }
+
+ /**
+ * Get the version of the blob cached locally. If the version is unknown or it has not been downloaded NOT_DOWNLOADED_VERSION should be
+ * returned. PRECONDITION: this can only be called with a lock on this instance held.
+ */
+ public abstract long getLocalVersion();
+
+ /**
+ * Get the version of the blob in the blob store. PRECONDITION: this can only be called with a lock on this instance held.
+ */
+ public abstract long getRemoteVersion(ClientBlobStore store) throws KeyNotFoundException, AuthorizationException;
+
+ /**
+ * Download the latest version to a temp location. This may also include unzipping some or all of the data to a temp location.
+ * PRECONDITION: this can only be called with a lock on this instance held.
+ *
+ * @param store the store to us to download the data.
+ * @return the version that was downloaded.
+ */
+ public abstract long downloadToTempLocation(ClientBlobStore store) throws IOException, KeyNotFoundException, AuthorizationException;
+
+ /**
* Commit the new version and make it available for the end user.
* PRECONDITION: uncompressToTempLocationIfNeeded will have been called.
* PRECONDITION: this can only be called with a lock on this instance held.
@@ -145,31 +163,6 @@ public abstract class LocallyCachedBlob {
public abstract long getSizeOnDisk();
/**
- * Get the size of p in bytes.
- * @param p the path to read.
- * @return the size of p in bytes.
- */
- protected static long getSizeOnDisk(Path p) throws IOException {
- if (!Files.exists(p)) {
- return 0;
- } else if (Files.isRegularFile(p)) {
- return Files.size(p);
- } else {
- //We will not follow sym links
- return Files.walk(p)
- .filter((subp) -> Files.isRegularFile(subp, LinkOption.NOFOLLOW_LINKS))
- .mapToLong((subp) -> {
- try {
- return Files.size(subp);
- } catch (IOException e) {
- LOG.warn("Could not get the size of ");
- }
- return 0;
- }).sum();
- }
- }
-
- /**
* Updates the last updated time. This should be called when references are added or removed.
*/
protected synchronized void touch() {
[02/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
index 2867e6a..2889cd1 100644
--- a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
+++ b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
@@ -1,26 +1,16 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.daemon.supervisor;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.lessThan;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+package org.apache.storm.daemon.supervisor;
import java.util.ArrayList;
import java.util.Collections;
@@ -29,11 +19,10 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
-
-import org.apache.storm.daemon.supervisor.Slot.StaticState;
-import org.apache.storm.daemon.supervisor.Slot.TopoProfileAction;
import org.apache.storm.daemon.supervisor.Slot.DynamicState;
import org.apache.storm.daemon.supervisor.Slot.MachineState;
+import org.apache.storm.daemon.supervisor.Slot.StaticState;
+import org.apache.storm.daemon.supervisor.Slot.TopoProfileAction;
import org.apache.storm.generated.ExecutorInfo;
import org.apache.storm.generated.LSWorkerHeartbeat;
import org.apache.storm.generated.LocalAssignment;
@@ -53,6 +42,23 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
public class SlotTest {
private static final Logger LOG = LoggerFactory.getLogger(SlotTest.class);
@@ -61,17 +67,17 @@ public class SlotTest {
if (cpu != null) {
resources.set_cpu(cpu);
}
-
+
if (mem_on_heap != null) {
resources.set_mem_on_heap(mem_on_heap);
}
-
+
if (mem_off_heap != null) {
resources.set_mem_off_heap(mem_off_heap);
}
return resources;
}
-
+
static LSWorkerHeartbeat mkWorkerHB(String id, int port, List<ExecutorInfo> exec, Integer timeSecs) {
LSWorkerHeartbeat ret = new LSWorkerHeartbeat();
ret.set_topology_id(id);
@@ -80,8 +86,8 @@ public class SlotTest {
ret.set_time_secs(timeSecs);
return ret;
}
-
- static List<ExecutorInfo> mkExecutorInfoList(int ... executors) {
+
+ static List<ExecutorInfo> mkExecutorInfoList(int... executors) {
ArrayList<ExecutorInfo> ret = new ArrayList<>(executors.length);
for (int exec : executors) {
ExecutorInfo execInfo = new ExecutorInfo();
@@ -91,7 +97,7 @@ public class SlotTest {
}
return ret;
}
-
+
static LocalAssignment mkLocalAssignment(String id, List<ExecutorInfo> exec, WorkerResources resources) {
LocalAssignment ret = new LocalAssignment();
ret.set_topology_id(id);
@@ -101,19 +107,19 @@ public class SlotTest {
}
return ret;
}
-
+
@Test
public void testEquivilant() {
- LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 100.0, 100.0));
- LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 200.0, 100.0));
- LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1,2,3,4,5,6), mkWorkerResources(100.0, 100.0, 100.0));
- LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6,5,4,3,2,1), mkWorkerResources(100.0, 100.0, 100.0));
+ LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), mkWorkerResources(100.0, 100.0, 100.0));
+ LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), mkWorkerResources(100.0, 200.0, 100.0));
+ LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1, 2, 3, 4, 5, 6), mkWorkerResources(100.0, 100.0, 100.0));
+ LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6, 5, 4, 3, 2, 1), mkWorkerResources(100.0, 100.0, 100.0));
assertTrue(Slot.equivalent(null, null));
assertTrue(Slot.equivalent(a, a));
assertTrue(Slot.equivalent(b, bReordered));
assertTrue(Slot.equivalent(bReordered, b));
-
+
assertFalse(Slot.equivalent(a, aResized));
assertFalse(Slot.equivalent(aResized, a));
assertFalse(Slot.equivalent(a, null));
@@ -123,10 +129,10 @@ public class SlotTest {
@Test
public void testForSameTopology() {
- LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 100.0, 100.0));
- LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 200.0, 100.0));
- LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1,2,3,4,5,6), mkWorkerResources(100.0, 100.0, 100.0));
- LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6,5,4,3,2,1), mkWorkerResources(100.0, 100.0, 100.0));
+ LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), mkWorkerResources(100.0, 100.0, 100.0));
+ LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1, 2, 3, 4, 5), mkWorkerResources(100.0, 200.0, 100.0));
+ LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1, 2, 3, 4, 5, 6), mkWorkerResources(100.0, 100.0, 100.0));
+ LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6, 5, 4, 3, 2, 1), mkWorkerResources(100.0, 100.0, 100.0));
assertTrue(Slot.forSameTopology(null, null));
assertTrue(Slot.forSameTopology(a, a));
@@ -142,29 +148,29 @@ public class SlotTest {
@Test
public void testEmptyToEmpty() throws Exception {
- try (SimulatedTime t = new SimulatedTime(1010)){
+ try (SimulatedTime t = new SimulatedTime(1010)) {
AsyncLocalizer localizer = mock(AsyncLocalizer.class);
LocalState state = mock(LocalState.class);
BlobChangingCallback cb = mock(BlobChangingCallback.class);
ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
ISupervisor iSuper = mock(ISupervisor.class);
StaticState staticState = new StaticState(localizer, 1000, 1000, 1000, 1000,
- containerLauncher, "localhost", 8080, iSuper, state, cb, null, null);
+ containerLauncher, "localhost", 8080, iSuper, state, cb, null, null);
DynamicState dynamicState = new DynamicState(null, null, null);
DynamicState nextState = Slot.handleEmpty(dynamicState, staticState);
assertEquals(MachineState.EMPTY, nextState.state);
assertTrue(Time.currentTimeMillis() > 1000);
}
}
-
+
@Test
public void testLaunchContainerFromEmpty() throws Exception {
- try (SimulatedTime t = new SimulatedTime(1010)){
+ try (SimulatedTime t = new SimulatedTime(1010)) {
int port = 8080;
String topoId = "NEW";
- List<ExecutorInfo> execList = mkExecutorInfoList(1,2,3,4,5);
- LocalAssignment newAssignment =
- mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
+ List<ExecutorInfo> execList = mkExecutorInfoList(1, 2, 3, 4, 5);
+ LocalAssignment newAssignment =
+ mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
AsyncLocalizer localizer = mock(AsyncLocalizer.class);
BlobChangingCallback cb = mock(BlobChangingCallback.class);
@@ -178,12 +184,12 @@ public class SlotTest {
@SuppressWarnings("unchecked")
CompletableFuture<Void> blobFuture = mock(CompletableFuture.class);
when(localizer.requestDownloadTopologyBlobs(newAssignment, port, cb)).thenReturn(blobFuture);
-
+
ISupervisor iSuper = mock(ISupervisor.class);
StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
- containerLauncher, "localhost", port, iSuper, state, cb, null, null);
+ containerLauncher, "localhost", port, iSuper, state, cb, null, null);
DynamicState dynamicState = new DynamicState(null, null, null)
- .withNewAssignment(newAssignment);
+ .withNewAssignment(newAssignment);
DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
verify(localizer).requestDownloadTopologyBlobs(newAssignment, port, cb);
@@ -191,7 +197,7 @@ public class SlotTest {
assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
assertEquals(newAssignment, nextState.pendingLocalization);
assertEquals(0, Time.currentTimeMillis());
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
verify(containerLauncher).launchContainer(port, newAssignment, state);
@@ -201,7 +207,7 @@ public class SlotTest {
assertSame(newAssignment, nextState.currentAssignment);
assertSame(container, nextState.container);
assertEquals(0, Time.currentTimeMillis());
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
assertSame("pendingDownload is not null", null, nextState.pendingDownload);
@@ -209,7 +215,7 @@ public class SlotTest {
assertSame(newAssignment, nextState.currentAssignment);
assertSame(container, nextState.container);
assertEquals(0, Time.currentTimeMillis());
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
assertSame("pendingDownload is not null", null, nextState.pendingDownload);
@@ -217,7 +223,7 @@ public class SlotTest {
assertSame(newAssignment, nextState.currentAssignment);
assertSame(container, nextState.container);
assertTrue(Time.currentTimeMillis() > 1000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
assertSame("pendingDownload is not null", null, nextState.pendingDownload);
@@ -231,59 +237,59 @@ public class SlotTest {
@Test
public void testRelaunch() throws Exception {
- try (SimulatedTime t = new SimulatedTime(1010)){
+ try (SimulatedTime t = new SimulatedTime(1010)) {
int port = 8080;
String topoId = "CURRENT";
- List<ExecutorInfo> execList = mkExecutorInfoList(1,2,3,4,5);
- LocalAssignment assignment =
- mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
-
+ List<ExecutorInfo> execList = mkExecutorInfoList(1, 2, 3, 4, 5);
+ LocalAssignment assignment =
+ mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
+
AsyncLocalizer localizer = mock(AsyncLocalizer.class);
BlobChangingCallback cb = mock(BlobChangingCallback.class);
Container container = mock(Container.class);
ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
- LSWorkerHeartbeat oldhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs()-10);
+ LSWorkerHeartbeat oldhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs() - 10);
LSWorkerHeartbeat goodhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs());
when(container.readHeartbeat()).thenReturn(oldhb, oldhb, goodhb, goodhb);
when(container.areAllProcessesDead()).thenReturn(false, true);
-
+
ISupervisor iSuper = mock(ISupervisor.class);
LocalState state = mock(LocalState.class);
StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
- containerLauncher, "localhost", port, iSuper, state, cb, null, null);
+ containerLauncher, "localhost", port, iSuper, state, cb, null, null);
DynamicState dynamicState = new DynamicState(assignment, container, assignment);
-
+
DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
assertEquals(MachineState.KILL_AND_RELAUNCH, nextState.state);
verify(container).kill();
assertTrue(Time.currentTimeMillis() > 1000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.KILL_AND_RELAUNCH, nextState.state);
verify(container).forceKill();
assertTrue(Time.currentTimeMillis() > 2000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
verify(container).relaunch();
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
assertTrue(Time.currentTimeMillis() > 3000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
}
}
-
+
@Test
public void testReschedule() throws Exception {
- try (SimulatedTime t = new SimulatedTime(1010)){
+ try (SimulatedTime t = new SimulatedTime(1010)) {
int port = 8080;
String cTopoId = "CURRENT";
- List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
- LocalAssignment cAssignment =
- mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
+ List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
+ LocalAssignment cAssignment =
+ mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
BlobChangingCallback cb = mock(BlobChangingCallback.class);
@@ -291,12 +297,12 @@ public class SlotTest {
LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
when(cContainer.readHeartbeat()).thenReturn(chb);
when(cContainer.areAllProcessesDead()).thenReturn(false, true);
-
+
String nTopoId = "NEW";
- List<ExecutorInfo> nExecList = mkExecutorInfoList(1,2,3,4,5);
- LocalAssignment nAssignment =
- mkLocalAssignment(nTopoId, nExecList, mkWorkerResources(100.0, 100.0, 100.0));
-
+ List<ExecutorInfo> nExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
+ LocalAssignment nAssignment =
+ mkLocalAssignment(nTopoId, nExecList, mkWorkerResources(100.0, 100.0, 100.0));
+
AsyncLocalizer localizer = mock(AsyncLocalizer.class);
Container nContainer = mock(Container.class);
LocalState state = mock(LocalState.class);
@@ -304,16 +310,16 @@ public class SlotTest {
when(containerLauncher.launchContainer(port, nAssignment, state)).thenReturn(nContainer);
LSWorkerHeartbeat nhb = mkWorkerHB(nTopoId, 100, nExecList, Time.currentTimeSecs());
when(nContainer.readHeartbeat()).thenReturn(nhb, nhb);
-
+
@SuppressWarnings("unchecked")
CompletableFuture<Void> blobFuture = mock(CompletableFuture.class);
when(localizer.requestDownloadTopologyBlobs(nAssignment, port, cb)).thenReturn(blobFuture);
-
+
ISupervisor iSuper = mock(ISupervisor.class);
StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
- containerLauncher, "localhost", port, iSuper, state, cb, null, null);
+ containerLauncher, "localhost", port, iSuper, state, cb, null, null);
DynamicState dynamicState = new DynamicState(cAssignment, cContainer, nAssignment);
-
+
DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
assertEquals(MachineState.KILL, nextState.state);
verify(cContainer).kill();
@@ -321,20 +327,20 @@ public class SlotTest {
assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
assertEquals(nAssignment, nextState.pendingLocalization);
assertTrue(Time.currentTimeMillis() > 1000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.KILL, nextState.state);
verify(cContainer).forceKill();
assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
assertEquals(nAssignment, nextState.pendingLocalization);
assertTrue(Time.currentTimeMillis() > 2000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
verify(cContainer).cleanUp();
verify(localizer).releaseSlotFor(cAssignment, port);
assertTrue(Time.currentTimeMillis() > 2000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
verify(containerLauncher).launchContainer(port, nAssignment, state);
@@ -344,7 +350,7 @@ public class SlotTest {
assertSame(nAssignment, nextState.currentAssignment);
assertSame(nContainer, nextState.container);
assertTrue(Time.currentTimeMillis() > 2000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
assertSame("pendingDownload is not null", null, nextState.pendingDownload);
@@ -352,7 +358,7 @@ public class SlotTest {
assertSame(nAssignment, nextState.currentAssignment);
assertSame(nContainer, nextState.container);
assertTrue(Time.currentTimeMillis() > 2000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
assertSame("pendingDownload is not null", null, nextState.pendingDownload);
@@ -360,7 +366,7 @@ public class SlotTest {
assertSame(nAssignment, nextState.currentAssignment);
assertSame(nContainer, nextState.container);
assertTrue(Time.currentTimeMillis() > 3000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
assertSame("pendingDownload is not null", null, nextState.pendingDownload);
@@ -370,31 +376,31 @@ public class SlotTest {
assertTrue(Time.currentTimeMillis() > 4000);
}
}
-
+
@Test
public void testRunningToEmpty() throws Exception {
- try (SimulatedTime t = new SimulatedTime(1010)){
+ try (SimulatedTime t = new SimulatedTime(1010)) {
int port = 8080;
String cTopoId = "CURRENT";
- List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
- LocalAssignment cAssignment =
- mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
-
+ List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
+ LocalAssignment cAssignment =
+ mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
+
Container cContainer = mock(Container.class);
LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
when(cContainer.readHeartbeat()).thenReturn(chb);
when(cContainer.areAllProcessesDead()).thenReturn(false, true);
-
+
AsyncLocalizer localizer = mock(AsyncLocalizer.class);
BlobChangingCallback cb = mock(BlobChangingCallback.class);
ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
-
+
ISupervisor iSuper = mock(ISupervisor.class);
LocalState state = mock(LocalState.class);
StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
- containerLauncher, "localhost", port, iSuper, state, cb, null, null);
+ containerLauncher, "localhost", port, iSuper, state, cb, null, null);
DynamicState dynamicState = new DynamicState(cAssignment, cContainer, null);
-
+
DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
assertEquals(MachineState.KILL, nextState.state);
verify(cContainer).kill();
@@ -402,14 +408,14 @@ public class SlotTest {
assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
assertEquals(null, nextState.pendingLocalization);
assertTrue(Time.currentTimeMillis() > 1000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.KILL, nextState.state);
verify(cContainer).forceKill();
assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
assertEquals(null, nextState.pendingLocalization);
assertTrue(Time.currentTimeMillis() > 2000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.EMPTY, nextState.state);
verify(cContainer).cleanUp();
@@ -417,13 +423,13 @@ public class SlotTest {
assertEquals(null, nextState.container);
assertEquals(null, nextState.currentAssignment);
assertTrue(Time.currentTimeMillis() > 2000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.EMPTY, nextState.state);
assertEquals(null, nextState.container);
assertEquals(null, nextState.currentAssignment);
assertTrue(Time.currentTimeMillis() > 3000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.EMPTY, nextState.state);
assertEquals(null, nextState.container);
@@ -431,29 +437,29 @@ public class SlotTest {
assertTrue(Time.currentTimeMillis() > 3000);
}
}
-
+
@Test
public void testRunWithProfileActions() throws Exception {
- try (SimulatedTime t = new SimulatedTime(1010)){
+ try (SimulatedTime t = new SimulatedTime(1010)) {
int port = 8080;
String cTopoId = "CURRENT";
- List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
- LocalAssignment cAssignment =
- mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
-
+ List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
+ LocalAssignment cAssignment =
+ mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
+
Container cContainer = mock(Container.class);
- LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs()+100); //NOT going to timeout for a while
+ LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs() + 100); //NOT going to timeout for a while
when(cContainer.readHeartbeat()).thenReturn(chb, chb, chb, chb, chb, chb);
when(cContainer.runProfiling(any(ProfileRequest.class), anyBoolean())).thenReturn(true);
-
+
AsyncLocalizer localizer = mock(AsyncLocalizer.class);
BlobChangingCallback cb = mock(BlobChangingCallback.class);
ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
-
+
ISupervisor iSuper = mock(ISupervisor.class);
LocalState state = mock(LocalState.class);
StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
- containerLauncher, "localhost", port, iSuper, state, cb, null, null);
+ containerLauncher, "localhost", port, iSuper, state, cb, null, null);
Set<TopoProfileAction> profileActions = new HashSet<>();
ProfileRequest request = new ProfileRequest();
request.set_action(ProfileAction.JPROFILE_STOP);
@@ -462,56 +468,57 @@ public class SlotTest {
info.add_to_port(port);
request.set_nodeInfo(info);
request.set_time_stamp(Time.currentTimeMillis() + 3000);//3 seconds from now
-
+
TopoProfileAction profile = new TopoProfileAction(cTopoId, request);
profileActions.add(profile);
Set<TopoProfileAction> expectedPending = new HashSet<>();
expectedPending.add(profile);
-
-
- DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment).withProfileActions(profileActions, Collections.<TopoProfileAction> emptySet());
-
+
+
+ DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment)
+ .withProfileActions(profileActions, Collections.<TopoProfileAction>emptySet());
+
DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
verify(cContainer).runProfiling(request, false);
assertEquals(expectedPending, nextState.pendingStopProfileActions);
assertEquals(expectedPending, nextState.profileActions);
assertTrue(Time.currentTimeMillis() > 1000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
assertEquals(expectedPending, nextState.pendingStopProfileActions);
assertEquals(expectedPending, nextState.profileActions);
assertTrue(Time.currentTimeMillis() > 2000);
-
-
+
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
assertEquals(expectedPending, nextState.pendingStopProfileActions);
assertEquals(expectedPending, nextState.profileActions);
assertTrue(Time.currentTimeMillis() > 3000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
verify(cContainer).runProfiling(request, true);
- assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.pendingStopProfileActions);
- assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.profileActions);
+ assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.pendingStopProfileActions);
+ assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.profileActions);
assertTrue(Time.currentTimeMillis() > 4000);
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
- assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.pendingStopProfileActions);
- assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.profileActions);
+ assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.pendingStopProfileActions);
+ assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.profileActions);
assertTrue(Time.currentTimeMillis() > 5000);
}
}
@Test
public void testResourcesChanged() throws Exception {
- try (SimulatedTime t = new SimulatedTime(1010)){
+ try (SimulatedTime t = new SimulatedTime(1010)) {
int port = 8080;
String cTopoId = "CURRENT";
- List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
+ List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
LocalAssignment cAssignment =
mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
@@ -531,7 +538,7 @@ public class SlotTest {
ISupervisor iSuper = mock(ISupervisor.class);
long heartbeatTimeoutMs = 5000;
StaticState staticState = new StaticState(localizer, heartbeatTimeoutMs, 120_000, 1000, 1000,
- containerLauncher, "localhost", port, iSuper, state, cb, null, null);
+ containerLauncher, "localhost", port, iSuper, state, cb, null, null);
Set<Slot.BlobChanging> changing = new HashSet<>();
LocallyCachedBlob stormJar = mock(LocallyCachedBlob.class);
@@ -541,7 +548,7 @@ public class SlotTest {
changing.add(new Slot.BlobChanging(cAssignment, stormJar, stormJarLatch));
DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment).withChangingBlobs(changing);
-
+
DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
assertEquals(MachineState.KILL_BLOB_UPDATE, nextState.state);
verify(iSuper).killedWorker(port);
@@ -579,7 +586,7 @@ public class SlotTest {
assertSame(nContainer, nextState.container);
assertThat(Time.currentTimeMillis(), greaterThan(2000L));
assertThat(Time.currentTimeMillis(), lessThan(heartbeatTimeoutMs));
-
+
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.RUNNING, nextState.state);
assertNull(nextState.pendingChangingBlobsAssignment);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/localizer/AsyncLocalizerTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/localizer/AsyncLocalizerTest.java b/storm-server/src/test/java/org/apache/storm/localizer/AsyncLocalizerTest.java
index 00a3f98..0e249b6 100644
--- a/storm-server/src/test/java/org/apache/storm/localizer/AsyncLocalizerTest.java
+++ b/storm-server/src/test/java/org/apache/storm/localizer/AsyncLocalizerTest.java
@@ -1,40 +1,17 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.localizer;
-import static org.apache.storm.blobstore.BlobStoreAclHandler.WORLD_EVERYTHING;
-import static org.apache.storm.localizer.LocalizedResource.USERCACHE;
-import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.endsWith;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.ArgumentMatchers.startsWith;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
import com.google.common.base.Joiner;
import java.io.File;
import java.io.FileInputStream;
@@ -55,41 +32,61 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
import org.apache.storm.blobstore.BlobStoreAclHandler;
+import org.apache.storm.blobstore.ClientBlobStore;
import org.apache.storm.blobstore.InputStreamWithMeta;
import org.apache.storm.blobstore.LocalFsBlobStore;
import org.apache.storm.daemon.supervisor.AdvancedFSOps;
import org.apache.storm.generated.AccessControl;
import org.apache.storm.generated.AccessControlType;
import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.ExecutorInfo;
import org.apache.storm.generated.KeyNotFoundException;
+import org.apache.storm.generated.LocalAssignment;
import org.apache.storm.generated.ReadableBlobMeta;
import org.apache.storm.generated.SettableBlobMeta;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.security.auth.DefaultPrincipalToLocal;
import org.apache.storm.utils.ConfigUtils;
-import org.apache.storm.utils.ServerUtils;
import org.apache.storm.utils.ReflectionUtils;
+import org.apache.storm.utils.ServerUtils;
import org.apache.storm.utils.Time;
import org.apache.storm.utils.Utils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-
-import org.apache.storm.Config;
-import org.apache.storm.blobstore.ClientBlobStore;
-import org.apache.storm.generated.ExecutorInfo;
-import org.apache.storm.generated.LocalAssignment;
-import org.apache.storm.generated.StormTopology;
-import org.apache.storm.security.auth.DefaultPrincipalToLocal;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.storm.blobstore.BlobStoreAclHandler.WORLD_EVERYTHING;
+import static org.apache.storm.localizer.LocalizedResource.USERCACHE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
public class AsyncLocalizerTest {
private static final Logger LOG = LoggerFactory.getLogger(AsyncLocalizerTest.class);
+ private final String user1 = "user1";
+ private final String user2 = "user2";
+ private final String user3 = "user3";
+ //From LocalizerTest
+ private File baseDir;
+ private ClientBlobStore mockblobstore = mock(ClientBlobStore.class);
private static String getTestLocalizerRoot() {
File f = new File("./target/" + Thread.currentThread().getStackTrace()[2].getMethodName() + "/localizer/");
@@ -187,25 +184,25 @@ public class AsyncLocalizerTest {
final int port = 8080;
final String simpleLocalName = "simple.txt";
final String simpleKey = "simple";
-
+
final String stormLocal = "/tmp/storm-local/";
final File userDir = new File(stormLocal, user);
- final String stormRoot = stormLocal+topoId+"/";
-
+ final String stormRoot = stormLocal + topoId + "/";
+
final String localizerRoot = getTestLocalizerRoot();
final String simpleCurrentLocalFile = localizerRoot + "/usercache/" + user + "/filecache/files/simple.current";
-
+
final StormTopology st = new StormTopology();
st.set_spouts(new HashMap<>());
st.set_bolts(new HashMap<>());
st.set_state_spouts(new HashMap<>());
-
+
Map<String, Map<String, Object>> topoBlobMap = new HashMap<>();
Map<String, Object> simple = new HashMap<>();
simple.put("localname", simpleLocalName);
simple.put("uncompress", false);
topoBlobMap.put(simpleKey, simple);
-
+
Map<String, Object> conf = new HashMap<>();
conf.put(Config.STORM_LOCAL_DIR, stormLocal);
AdvancedFSOps ops = mock(AdvancedFSOps.class);
@@ -214,7 +211,7 @@ public class AsyncLocalizerTest {
Map<String, Object> topoConf = new HashMap<>(conf);
topoConf.put(Config.TOPOLOGY_BLOBSTORE_MAP, topoBlobMap);
topoConf.put(Config.TOPOLOGY_NAME, topoName);
-
+
List<LocalizedResource> localizedList = new ArrayList<>();
LocalizedResource simpleLocal = new LocalizedResource(simpleKey, Paths.get(localizerRoot), false, ops, conf, user);
localizedList.add(simpleLocal);
@@ -228,7 +225,7 @@ public class AsyncLocalizerTest {
//Write the mocking backwards so the actual method is not called on the spy object
doReturn(CompletableFuture.supplyAsync(() -> null)).when(bl)
- .requestDownloadBaseTopologyBlobs(any(), eq(null));
+ .requestDownloadBaseTopologyBlobs(any(), eq(null));
doReturn(userDir).when(bl).getLocalUserFileCacheDir(user);
doReturn(localizedList).when(bl).getBlobs(any(List.class), any(), any());
@@ -254,119 +251,9 @@ public class AsyncLocalizerTest {
}
}
-
- //From LocalizerTest
- private File baseDir;
-
- private final String user1 = "user1";
- private final String user2 = "user2";
- private final String user3 = "user3";
-
- private ClientBlobStore mockblobstore = mock(ClientBlobStore.class);
-
-
- class TestLocalizer extends AsyncLocalizer {
-
- TestLocalizer(Map<String, Object> conf, String baseDir) throws IOException {
- super(conf, AdvancedFSOps.make(conf), baseDir);
- }
-
- @Override
- protected ClientBlobStore getClientBlobStore() {
- return mockblobstore;
- }
-
- synchronized void addReferences(List<LocalResource> localresource, PortAndAssignment pna, BlobChangingCallback cb) {
- String user = pna.getOwner();
- for (LocalResource blob : localresource) {
- ConcurrentMap<String, LocalizedResource> lrsrcSet = blob.shouldUncompress() ? userArchives.get(user) : userFiles.get(user);
- if (lrsrcSet != null) {
- LocalizedResource lrsrc = lrsrcSet.get(blob.getBlobName());
- if (lrsrc != null) {
- lrsrc.addReference(pna, blob.needsCallback() ? cb : null);
- LOG.debug("added reference for topo: {} key: {}", pna, blob);
- } else {
- LOG.warn("trying to add reference to non-existent blob, key: {} topo: {}", blob, pna);
- }
- } else {
- LOG.warn("trying to add reference to non-existent local resource set, user: {} topo: {}", user, pna);
- }
- }
- }
-
- void setTargetCacheSize(long size) {
- cacheTargetSize = size;
- }
-
- // For testing, be careful as it doesn't clone
- ConcurrentHashMap<String, ConcurrentHashMap<String, LocalizedResource>> getUserFiles() {
- return userFiles;
- }
-
- ConcurrentHashMap<String, ConcurrentHashMap<String, LocalizedResource>> getUserArchives() {
- return userArchives;
- }
-
- /**
- * This function either returns the blob in the existing cache or if it doesn't exist in the
- * cache, it will download the blob and will block until the download is complete.
- */
- LocalizedResource getBlob(LocalResource localResource, PortAndAssignment pna, BlobChangingCallback cb)
- throws AuthorizationException, KeyNotFoundException, IOException {
- ArrayList<LocalResource> arr = new ArrayList<>();
- arr.add(localResource);
- List<LocalizedResource> results = getBlobs(arr, pna, cb);
- if (results.isEmpty() || results.size() != 1) {
- throw new IOException("Unknown error getting blob: " + localResource + ", for user: " + pna.getOwner() +
- ", topo: " + pna);
- }
- return results.get(0);
- }
- }
-
- class TestInputStreamWithMeta extends InputStreamWithMeta {
- private final long version;
- private InputStream iostream;
-
- public TestInputStreamWithMeta(long version) {
- iostream = IOUtils.toInputStream("some test data for my input stream");
- this.version = version;
- }
-
- public TestInputStreamWithMeta(InputStream istream, long version) {
- iostream = istream;
- this.version = version;
- }
-
- @Override
- public long getVersion() throws IOException {
- return version;
- }
-
- @Override
- public synchronized int read() {
- return 0;
- }
-
- @Override
- public synchronized int read(byte[] b)
- throws IOException {
- int length = iostream.read(b);
- if (length == 0) {
- return -1;
- }
- return length;
- }
-
- @Override
- public long getFileLength() {
- return 0;
- }
- };
-
@Before
public void setUp() throws Exception {
- baseDir = new File(System.getProperty("java.io.tmpdir") + "/blob-store-localizer-test-"+ UUID.randomUUID());
+ baseDir = new File(System.getProperty("java.io.tmpdir") + "/blob-store-localizer-test-" + UUID.randomUUID());
if (!baseDir.mkdir()) {
throw new IOException("failed to create base directory");
}
@@ -376,9 +263,12 @@ public class AsyncLocalizerTest {
public void tearDown() throws Exception {
try {
FileUtils.deleteDirectory(baseDir);
- } catch (IOException ignore) {}
+ } catch (IOException ignore) {
+ }
}
+ ;
+
protected String joinPath(String... pathList) {
return Joiner.on(File.separator).join(pathList);
}
@@ -402,11 +292,11 @@ public class AsyncLocalizerTest {
String expectedDir = constructUserCacheDir(baseDir.toString(), user1);
assertEquals("get local user dir doesn't return right value",
- expectedDir, localizer.getLocalUserDir(user1).toString());
+ expectedDir, localizer.getLocalUserDir(user1).toString());
String expectedFileDir = joinPath(expectedDir, LocalizedResource.FILECACHE);
assertEquals("get local user file dir doesn't return right value",
- expectedFileDir, localizer.getLocalUserFileCacheDir(user1).toString());
+ expectedFileDir, localizer.getLocalUserFileCacheDir(user1).toString());
}
@Test
@@ -545,7 +435,8 @@ public class AsyncLocalizerTest {
when(mockblobstore.getBlobMeta(key1)).thenReturn(rbm);
when(mockblobstore.getBlob(key1)).thenReturn(new TestInputStreamWithMeta(new
- FileInputStream(archiveFile.getAbsolutePath()), 0));
+ FileInputStream(archiveFile.getAbsolutePath()),
+ 0));
long timeBefore = Time.currentTimeMillis();
Time.advanceTime(10);
@@ -583,7 +474,7 @@ public class AsyncLocalizerTest {
assertEquals("file path doesn't match", keyFile.toPath(), key1rsrc.getFilePathWithVersion());
assertEquals("size doesn't match", size, key1rsrc.getSizeOnDisk());
assertTrue("timestamp not within range", (key1rsrc.getLastUsed() >= timeBefore && key1rsrc
- .getLastUsed() <= timeAfter));
+ .getLastUsed() <= timeAfter));
timeBefore = Time.currentTimeMillis();
Time.advanceTime(10);
@@ -598,7 +489,7 @@ public class AsyncLocalizerTest {
assertNotNull("Local resource doesn't exist but should", key1rsrc);
assertEquals("refcount doesn't match " + key1rsrc.getDependencies(), false, key1rsrc.isUsed());
assertTrue("timestamp not within range", (key1rsrc.getLastUsed() >= timeBefore && key1rsrc
- .getLastUsed() <= timeAfter));
+ .getLastUsed() <= timeAfter));
// should remove the blob since cache size set really small
localizer.cleanup();
@@ -660,7 +551,7 @@ public class AsyncLocalizerTest {
assertEquals("file path doesn't match", keyFile.toPath(), key1rsrc.getCurrentSymlinkPath());
assertEquals("size doesn't match", 34, key1rsrc.getSizeOnDisk());
assertTrue("timestamp not within range", (key1rsrc.getLastUsed() >= timeBefore && key1rsrc
- .getLastUsed() <= timeAfter));
+ .getLastUsed() <= timeAfter));
timeBefore = Time.currentTimeMillis();
Time.advanceTime(10);
@@ -675,7 +566,7 @@ public class AsyncLocalizerTest {
assertNotNull("Local resource doesn't exist but should", key1rsrc);
assertEquals("refcount doesn't match " + key1rsrc.getDependencies(), false, key1rsrc.isUsed());
assertTrue("timestamp not within range " + timeBefore + " " + key1rsrc.getLastUsed() + " " + timeAfter,
- (key1rsrc.getLastUsed() >= timeBefore && key1rsrc.getLastUsed() <= timeAfter));
+ (key1rsrc.getLastUsed() >= timeBefore && key1rsrc.getLastUsed() <= timeAfter));
// should remove the blob since cache size set really small
localizer.cleanup();
@@ -710,8 +601,10 @@ public class AsyncLocalizerTest {
when(mockblobstore.getBlob(key2)).thenReturn(new TestInputStreamWithMeta(0));
when(mockblobstore.getBlob(key3)).thenReturn(new TestInputStreamWithMeta(0));
- List<LocalResource> keys = Arrays.asList(new LocalResource[]{new LocalResource(key1, false, false),
- new LocalResource(key2, false, false), new LocalResource(key3, false, false)});
+ List<LocalResource> keys = Arrays.asList(new LocalResource[]{
+ new LocalResource(key1, false, false),
+ new LocalResource(key2, false, false), new LocalResource(key3, false, false)
+ });
File user1Dir = localizer.getLocalUserFileCacheDir(user1);
assertTrue("failed to create user dir", user1Dir.mkdirs());
@@ -724,7 +617,7 @@ public class AsyncLocalizerTest {
LocalizedResource lrsrc3 = lrsrcs.get(2);
String expectedFileDir = joinPath(baseDir.toString(), USERCACHE, user1,
- LocalizedResource.FILECACHE, LocalizedResource.FILESDIR);
+ LocalizedResource.FILECACHE, LocalizedResource.FILESDIR);
assertTrue("user filecache dir not created", new File(expectedFileDir).exists());
File keyFile = new File(expectedFileDir, key1 + LocalizedResource.CURRENT_BLOB_SUFFIX);
File keyFile2 = new File(expectedFileDir, key2 + LocalizedResource.CURRENT_BLOB_SUFFIX);
@@ -760,7 +653,7 @@ public class AsyncLocalizerTest {
lrsrc = localizer.getBlob(new LocalResource(key1, false, false), topo1Pna, null);
LOG.info("Got Blob...");
assertTrue("timestamp not within range " + timeBefore + " <= " + lrsrc.getLastUsed() + " <= " + timeAfter,
- (lrsrc.getLastUsed() >= timeBefore && lrsrc.getLastUsed() <= timeAfter));
+ (lrsrc.getLastUsed() >= timeBefore && lrsrc.getLastUsed() <= timeAfter));
//Resets the last access time for key1
localizer.removeBlobReference(lrsrc.getKey(), topo1Pna, false);
@@ -829,7 +722,7 @@ public class AsyncLocalizerTest {
LocalFsBlobStore spy = spy(bs);
Mockito.doReturn(true).when(spy).checkForBlobOrDownload(key1);
Mockito.doNothing().when(spy).checkForBlobUpdate(key1);
- spy.prepare(conf,null,null);
+ spy.prepare(conf, null, null);
spy.getBlob(key1, null);
}
@@ -837,7 +730,7 @@ public class AsyncLocalizerTest {
public void testMultipleUsers() throws Exception {
Map<String, Object> conf = new HashMap<>();
// set clean time really high so doesn't kick in
- conf.put(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, 60*60*1000);
+ conf.put(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, 60 * 60 * 1000);
String topo1 = "topo1";
String topo2 = "topo2";
@@ -884,9 +777,9 @@ public class AsyncLocalizerTest {
String expectedUserDir1 = joinPath(baseDir.toString(), USERCACHE, user1);
String expectedFileDirUser1 = joinPath(expectedUserDir1, LocalizedResource.FILECACHE, LocalizedResource.FILESDIR);
String expectedFileDirUser2 = joinPath(baseDir.toString(), USERCACHE, user2,
- LocalizedResource.FILECACHE, LocalizedResource.FILESDIR);
+ LocalizedResource.FILECACHE, LocalizedResource.FILESDIR);
String expectedFileDirUser3 = joinPath(baseDir.toString(), USERCACHE, user3,
- LocalizedResource.FILECACHE, LocalizedResource.FILESDIR);
+ LocalizedResource.FILECACHE, LocalizedResource.FILESDIR);
assertTrue("user filecache dir user1 not created", new File(expectedFileDirUser1).exists());
assertTrue("user filecache dir user2 not created", new File(expectedFileDirUser2).exists());
assertTrue("user filecache dir user3 not created", new File(expectedFileDirUser3).exists());
@@ -929,7 +822,7 @@ public class AsyncLocalizerTest {
public void testUpdate() throws Exception {
Map<String, Object> conf = new HashMap<>();
// set clean time really high so doesn't kick in
- conf.put(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, 60*60*1000);
+ conf.put(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, 60 * 60 * 1000);
String key1 = "key1";
String topo1 = "topo1";
@@ -983,6 +876,105 @@ public class AsyncLocalizerTest {
localizer.updateBlobs();
assertTrue("blob version file not created", versionFile.exists());
assertEquals("blob version not correct", 3, LocalizedResource.localVersionOfBlob(keyVersionFile));
- assertTrue("blob file with version 3 not created", new File(expectedFileDir, key1 + ".3").exists());
+ assertTrue("blob file with version 3 not created", new File(expectedFileDir, key1 + ".3").exists());
+ }
+
+ class TestLocalizer extends AsyncLocalizer {
+
+ TestLocalizer(Map<String, Object> conf, String baseDir) throws IOException {
+ super(conf, AdvancedFSOps.make(conf), baseDir);
+ }
+
+ @Override
+ protected ClientBlobStore getClientBlobStore() {
+ return mockblobstore;
+ }
+
+ synchronized void addReferences(List<LocalResource> localresource, PortAndAssignment pna, BlobChangingCallback cb) {
+ String user = pna.getOwner();
+ for (LocalResource blob : localresource) {
+ ConcurrentMap<String, LocalizedResource> lrsrcSet = blob.shouldUncompress() ? userArchives.get(user) : userFiles.get(user);
+ if (lrsrcSet != null) {
+ LocalizedResource lrsrc = lrsrcSet.get(blob.getBlobName());
+ if (lrsrc != null) {
+ lrsrc.addReference(pna, blob.needsCallback() ? cb : null);
+ LOG.debug("added reference for topo: {} key: {}", pna, blob);
+ } else {
+ LOG.warn("trying to add reference to non-existent blob, key: {} topo: {}", blob, pna);
+ }
+ } else {
+ LOG.warn("trying to add reference to non-existent local resource set, user: {} topo: {}", user, pna);
+ }
+ }
+ }
+
+ void setTargetCacheSize(long size) {
+ cacheTargetSize = size;
+ }
+
+ // For testing, be careful as it doesn't clone
+ ConcurrentHashMap<String, ConcurrentHashMap<String, LocalizedResource>> getUserFiles() {
+ return userFiles;
+ }
+
+ ConcurrentHashMap<String, ConcurrentHashMap<String, LocalizedResource>> getUserArchives() {
+ return userArchives;
+ }
+
+ /**
+ * This function either returns the blob in the existing cache or if it doesn't exist in the
+ * cache, it will download the blob and will block until the download is complete.
+ */
+ LocalizedResource getBlob(LocalResource localResource, PortAndAssignment pna, BlobChangingCallback cb)
+ throws AuthorizationException, KeyNotFoundException, IOException {
+ ArrayList<LocalResource> arr = new ArrayList<>();
+ arr.add(localResource);
+ List<LocalizedResource> results = getBlobs(arr, pna, cb);
+ if (results.isEmpty() || results.size() != 1) {
+ throw new IOException("Unknown error getting blob: " + localResource + ", for user: " + pna.getOwner() +
+ ", topo: " + pna);
+ }
+ return results.get(0);
+ }
+ }
+
+ class TestInputStreamWithMeta extends InputStreamWithMeta {
+ private final long version;
+ private InputStream iostream;
+
+ public TestInputStreamWithMeta(long version) {
+ iostream = IOUtils.toInputStream("some test data for my input stream");
+ this.version = version;
+ }
+
+ public TestInputStreamWithMeta(InputStream istream, long version) {
+ iostream = istream;
+ this.version = version;
+ }
+
+ @Override
+ public long getVersion() throws IOException {
+ return version;
+ }
+
+ @Override
+ public synchronized int read() {
+ return 0;
+ }
+
+ @Override
+ public synchronized int read(byte[] b)
+ throws IOException {
+ int length = iostream.read(b);
+ if (length == 0) {
+ return -1;
+ }
+ return length;
+ }
+
+ @Override
+ public long getFileLength() {
+ return 0;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java b/storm-server/src/test/java/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
index 4ebf5cf..a6e8bfd 100644
--- a/storm-server/src/test/java/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
+++ b/storm-server/src/test/java/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.localizer;
@@ -33,7 +28,9 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
public class LocalizedResourceRetentionSetTest {
@@ -76,7 +73,7 @@ public class LocalizedResourceRetentionSetTest {
@Test
public void testCleanup() throws Exception {
ClientBlobStore mockBlobstore = mock(ClientBlobStore.class);
- when (mockBlobstore.getBlobMeta(any())).thenReturn(new ReadableBlobMeta(new SettableBlobMeta(), 1));
+ when(mockBlobstore.getBlobMeta(any())).thenReturn(new ReadableBlobMeta(new SettableBlobMeta(), 1));
PortAndAssignment pna1 = new PortAndAssignment(1, new LocalAssignment("topo1", Collections.emptyList()));
String user = "user";
Map<String, Object> conf = new HashMap<>();
@@ -86,15 +83,15 @@ public class LocalizedResourceRetentionSetTest {
ConcurrentMap<String, LocalizedResource> lrArchives = new ConcurrentHashMap<>();
// no reference to key1
LocalizedResource localresource1 = new LocalizedResource("key1", Paths.get("./target/TESTING/testfile1"), false, ops, conf,
- user);
+ user);
localresource1.setSize(10);
// no reference to archive1
LocalizedResource archiveresource1 = new LocalizedResource("archive1", Paths.get("./target/TESTING/testarchive1"), true, ops,
- conf, user);
+ conf, user);
archiveresource1.setSize(20);
// reference to key2
LocalizedResource localresource2 = new LocalizedResource("key2", Paths.get("./target/TESTING/testfile2"), false, ops, conf,
- user);
+ user);
localresource2.addReference(pna1, null);
// check adding reference to local resource with topology of same name
localresource2.addReference(pna1, null);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/metric/ClusterMetricsConsumerExecutorTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/metric/ClusterMetricsConsumerExecutorTest.java b/storm-server/src/test/java/org/apache/storm/metric/ClusterMetricsConsumerExecutorTest.java
index 0d7b520..dcb6a03 100644
--- a/storm-server/src/test/java/org/apache/storm/metric/ClusterMetricsConsumerExecutorTest.java
+++ b/storm-server/src/test/java/org/apache/storm/metric/ClusterMetricsConsumerExecutorTest.java
@@ -1,32 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.metric;
+import java.util.Collection;
+import java.util.Collections;
import org.apache.storm.metric.api.DataPoint;
import org.apache.storm.metric.api.IClusterMetricsConsumer;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.Collection;
-import java.util.Collections;
-
import static org.mockito.Mockito.mock;
public class ClusterMetricsConsumerExecutorTest {
@@ -39,7 +32,7 @@ public class ClusterMetricsConsumerExecutorTest {
@Test
public void testPrepareDoesNotThrowExceptionWhenInitializingClusterMetricsConsumerIsFailing() throws Exception {
ClusterMetricsConsumerExecutor sut = new ClusterMetricsConsumerExecutor(
- MockFailingClusterMetricsConsumer.class.getName(), 2);
+ MockFailingClusterMetricsConsumer.class.getName(), 2);
// it shouldn't propagate any exceptions
sut.prepare();
@@ -51,7 +44,7 @@ public class ClusterMetricsConsumerExecutorTest {
@Test
public void testHandleDataPointsWithClusterMetricsShouldSkipHandlingMetricsIfFailedBefore() throws Exception {
ClusterMetricsConsumerExecutor sut = new ClusterMetricsConsumerExecutor(
- MockFailingClusterMetricsConsumer.class.getName(), 2);
+ MockFailingClusterMetricsConsumer.class.getName(), 2);
// below calls shouldn't propagate any exceptions
sut.prepare();
@@ -66,7 +59,7 @@ public class ClusterMetricsConsumerExecutorTest {
@Test
public void testHandleDataPointsWithSupervisorMetricsShouldRetryInitializingClusterMetricsConsumerIfFailedBefore() throws Exception {
ClusterMetricsConsumerExecutor sut = new ClusterMetricsConsumerExecutor(
- MockFailingClusterMetricsConsumer.class.getName(), 2);
+ MockFailingClusterMetricsConsumer.class.getName(), 2);
// below calls shouldn't propagate any exceptions
sut.prepare();
@@ -84,28 +77,6 @@ public class ClusterMetricsConsumerExecutorTest {
private static int handleDataPointsWithSupervisorInfoCallCount = 0;
private static int cleanupCallCount = 0;
- @Override
- public void prepare(Object registrationArgument) {
- prepareCallCount++;
-
- throw new RuntimeException("prepare failing...");
- }
-
- @Override
- public void handleDataPoints(ClusterInfo clusterInfo, Collection<DataPoint> dataPoints) {
- handleDataPointsWithClusterInfoCallCount++;
- }
-
- @Override
- public void handleDataPoints(SupervisorInfo supervisorInfo, Collection<DataPoint> dataPoints) {
- handleDataPointsWithSupervisorInfoCallCount++;
- }
-
- @Override
- public void cleanup() {
- cleanupCallCount++;
- }
-
public static int getPrepareCallCount() {
return prepareCallCount;
}
@@ -128,6 +99,28 @@ public class ClusterMetricsConsumerExecutorTest {
handleDataPointsWithSupervisorInfoCallCount = 0;
cleanupCallCount = 0;
}
+
+ @Override
+ public void prepare(Object registrationArgument) {
+ prepareCallCount++;
+
+ throw new RuntimeException("prepare failing...");
+ }
+
+ @Override
+ public void handleDataPoints(ClusterInfo clusterInfo, Collection<DataPoint> dataPoints) {
+ handleDataPointsWithClusterInfoCallCount++;
+ }
+
+ @Override
+ public void handleDataPoints(SupervisorInfo supervisorInfo, Collection<DataPoint> dataPoints) {
+ handleDataPointsWithSupervisorInfoCallCount++;
+ }
+
+ @Override
+ public void cleanup() {
+ cleanupCallCount++;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/nimbus/LocalNimbusTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/nimbus/LocalNimbusTest.java b/storm-server/src/test/java/org/apache/storm/nimbus/LocalNimbusTest.java
index 64306a2..916fff0 100644
--- a/storm-server/src/test/java/org/apache/storm/nimbus/LocalNimbusTest.java
+++ b/storm-server/src/test/java/org/apache/storm/nimbus/LocalNimbusTest.java
@@ -16,6 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.storm.nimbus;
import java.util.ArrayList;
@@ -42,6 +43,19 @@ import org.junit.Test;
*/
public class LocalNimbusTest {
+ public static StormTopology createTestTopology() {
+ TopologyBuilder builder = new TopologyBuilder();
+ builder.setSpout("words", new TestWordSpout(), generateParallelismHint());
+ builder.setBolt("count", new TestWordCounter(), generateParallelismHint()).shuffleGrouping("words");
+ builder.setBolt("globalCount", new TestGlobalCount(), generateParallelismHint()).shuffleGrouping("count");
+
+ return builder.createTopology();
+ }
+
+ private static int generateParallelismHint() {
+ return new Random().nextInt(9) + 1;
+ }
+
@Test
public void testSubmitTopologyToLocalNimbus() throws Exception {
int port = Utils.getAvailablePort();
@@ -68,19 +82,6 @@ public class LocalNimbusTest {
}
}
- public static StormTopology createTestTopology() {
- TopologyBuilder builder = new TopologyBuilder();
- builder.setSpout("words", new TestWordSpout(), generateParallelismHint());
- builder.setBolt("count", new TestWordCounter(), generateParallelismHint()).shuffleGrouping("words");
- builder.setBolt("globalCount", new TestGlobalCount(), generateParallelismHint()).shuffleGrouping("count");
-
- return builder.createTopology();
- }
-
- private static int generateParallelismHint() {
- return new Random().nextInt(9)+1;
- }
-
public static class InmemoryTopologySubmitterHook implements ISubmitterHook {
public static final List<TopologyDetails> submittedTopologies = new ArrayList<>();
@@ -106,8 +107,9 @@ public class LocalNimbusTest {
TopologyDetails that = (TopologyDetails) o;
- if (topologyName != null ? !topologyName.equals(that.topologyName) : that.topologyName != null)
+ if (topologyName != null ? !topologyName.equals(that.topologyName) : that.topologyName != null) {
return false;
+ }
return !(stormTopology != null ? !stormTopology.equals(that.stormTopology) : that.stormTopology != null);
}
@@ -122,9 +124,9 @@ public class LocalNimbusTest {
@Override
public String toString() {
return "TopologyDetails{" +
- "topologyName='" + topologyName + '\'' +
- ", stormTopology=" + stormTopology +
- '}';
+ "topologyName='" + topologyName + '\'' +
+ ", stormTopology=" + stormTopology +
+ '}';
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/test/java/org/apache/storm/scheduler/ClusterTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/scheduler/ClusterTest.java b/storm-server/src/test/java/org/apache/storm/scheduler/ClusterTest.java
index b568b04..3b9eb14 100644
--- a/storm-server/src/test/java/org/apache/storm/scheduler/ClusterTest.java
+++ b/storm-server/src/test/java/org/apache/storm/scheduler/ClusterTest.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.scheduler;
import java.util.HashMap;
import java.util.Map;
-
import org.apache.storm.Config;
import org.junit.Assert;
import org.junit.Test;
@@ -99,8 +93,8 @@ public class ClusterTest {
@Test
public void getAssignedMemoryForSlot_topologyWorkerLwChildopts() {
- singleValueTest(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS, "-Xmx64m",
- TOPOLOGY_WORKER_DEFAULT_MEMORY_ALLOCATION + 64.0);
+ singleValueTest(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS, "-Xmx64m",
+ TOPOLOGY_WORKER_DEFAULT_MEMORY_ALLOCATION + 64.0);
}
@Test
[49/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-hbase
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseMapState.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseMapState.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseMapState.java
index 9dbcd80..d2a73d4 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseMapState.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseMapState.java
@@ -19,14 +19,12 @@
package org.apache.storm.hbase.trident.state;
import com.google.common.collect.Maps;
-
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Get;
@@ -61,13 +59,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HBaseMapState<T> implements IBackingMap<T> {
- private static Logger LOG = LoggerFactory.getLogger(HBaseMapState.class);
-
- private int partitionNum;
-
-
@SuppressWarnings("rawtypes")
private static final Map<StateType, Serializer> DEFAULT_SERIALZERS = Maps.newHashMap();
+ private static Logger LOG = LoggerFactory.getLogger(HBaseMapState.class);
static {
DEFAULT_SERIALZERS.put(StateType.NON_TRANSACTIONAL, new JSONNonTransactionalSerializer());
@@ -75,6 +69,7 @@ public class HBaseMapState<T> implements IBackingMap<T> {
DEFAULT_SERIALZERS.put(StateType.OPAQUE, new JSONOpaqueSerializer());
}
+ private int partitionNum;
private Options<T> options;
private Serializer<T> serializer;
private HTable table;
@@ -82,8 +77,8 @@ public class HBaseMapState<T> implements IBackingMap<T> {
/**
* Constructor.
*
- * @param options HBase State options.
- * @param map topology config map.
+ * @param options HBase State options.
+ * @param map topology config map.
* @param partitionNum the number of partition.
*/
public HBaseMapState(final Options<T> options, Map<String, Object> map, int partitionNum) {
@@ -92,7 +87,7 @@ public class HBaseMapState<T> implements IBackingMap<T> {
this.partitionNum = partitionNum;
final Configuration hbConfig = HBaseConfiguration.create();
- Map<String, Object> conf = (Map<String, Object>)map.get(options.configKey);
+ Map<String, Object> conf = (Map<String, Object>) map.get(options.configKey);
if (conf == null) {
LOG.info("HBase configuration not found using key '" + options.configKey + "'");
LOG.info("Using HBase config from first hbase-site.xml found on classpath.");
@@ -114,19 +109,6 @@ public class HBaseMapState<T> implements IBackingMap<T> {
}
-
- public static class Options<T> implements Serializable {
-
- public Serializer<T> serializer = null;
- public int cacheSize = 5000;
- public String globalKey = "$HBASE_STATE_GLOBAL$";
- public String configKey = "hbase.config";
- public String tableName;
- public String columnFamily;
- public TridentHBaseMapMapper mapMapper;
- }
-
-
@SuppressWarnings("rawtypes")
public static StateFactory opaque() {
Options<OpaqueValue> options = new Options<OpaqueValue>();
@@ -159,57 +141,6 @@ public class HBaseMapState<T> implements IBackingMap<T> {
return new Factory(StateType.NON_TRANSACTIONAL, opts);
}
-
- protected static class Factory implements StateFactory {
- private StateType stateType;
- private Options options;
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- public Factory(StateType stateType, Options options) {
- this.stateType = stateType;
- this.options = options;
-
- if (this.options.serializer == null) {
- this.options.serializer = DEFAULT_SERIALZERS.get(stateType);
- }
-
- if (this.options.serializer == null) {
- throw new RuntimeException("Serializer should be specified for type: " + stateType);
- }
-
- if (this.options.mapMapper == null) {
- throw new RuntimeException("MapMapper should be specified for type: " + stateType);
- }
- }
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- public State makeState(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
- LOG.info("Preparing HBase State for partition {} of {}.", partitionIndex + 1, numPartitions);
- IBackingMap state = new HBaseMapState(options, conf, partitionIndex);
-
- if (options.cacheSize > 0) {
- state = new CachedMap(state, options.cacheSize);
- }
-
- MapState mapState;
- switch (stateType) {
- case NON_TRANSACTIONAL:
- mapState = NonTransactionalMap.build(state);
- break;
- case OPAQUE:
- mapState = OpaqueMap.build(state);
- break;
- case TRANSACTIONAL:
- mapState = TransactionalMap.build(state);
- break;
- default:
- throw new IllegalArgumentException("Unknown state type: " + stateType);
- }
- return new SnapshottableMap(mapState, new Values(options.globalKey));
- }
-
- }
-
@Override
public List<T> multiGet(List<List<Object>> keys) {
List<Get> gets = new ArrayList<Get>();
@@ -249,7 +180,7 @@ public class HBaseMapState<T> implements IBackingMap<T> {
byte[] hbaseKey = this.options.mapMapper.rowKey(keys.get(i));
String qualifier = this.options.mapMapper.qualifier(keys.get(i));
LOG.info("Partiton: {}, Key: {}, Value: {}",
- new Object[]{this.partitionNum, new String(hbaseKey), new String(this.serializer.serialize(values.get(i)))});
+ new Object[]{ this.partitionNum, new String(hbaseKey), new String(this.serializer.serialize(values.get(i))) });
Put put = new Put(hbaseKey);
T val = values.get(i);
put.add(this.options.columnFamily.getBytes(),
@@ -268,4 +199,65 @@ public class HBaseMapState<T> implements IBackingMap<T> {
throw new FailedException("IOException while writing to HBase", e);
}
}
+
+ public static class Options<T> implements Serializable {
+
+ public Serializer<T> serializer = null;
+ public int cacheSize = 5000;
+ public String globalKey = "$HBASE_STATE_GLOBAL$";
+ public String configKey = "hbase.config";
+ public String tableName;
+ public String columnFamily;
+ public TridentHBaseMapMapper mapMapper;
+ }
+
+ protected static class Factory implements StateFactory {
+ private StateType stateType;
+ private Options options;
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public Factory(StateType stateType, Options options) {
+ this.stateType = stateType;
+ this.options = options;
+
+ if (this.options.serializer == null) {
+ this.options.serializer = DEFAULT_SERIALZERS.get(stateType);
+ }
+
+ if (this.options.serializer == null) {
+ throw new RuntimeException("Serializer should be specified for type: " + stateType);
+ }
+
+ if (this.options.mapMapper == null) {
+ throw new RuntimeException("MapMapper should be specified for type: " + stateType);
+ }
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public State makeState(Map<String, Object> conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ LOG.info("Preparing HBase State for partition {} of {}.", partitionIndex + 1, numPartitions);
+ IBackingMap state = new HBaseMapState(options, conf, partitionIndex);
+
+ if (options.cacheSize > 0) {
+ state = new CachedMap(state, options.cacheSize);
+ }
+
+ MapState mapState;
+ switch (stateType) {
+ case NON_TRANSACTIONAL:
+ mapState = NonTransactionalMap.build(state);
+ break;
+ case OPAQUE:
+ mapState = OpaqueMap.build(state);
+ break;
+ case TRANSACTIONAL:
+ mapState = TransactionalMap.build(state);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown state type: " + stateType);
+ }
+ return new SnapshottableMap(mapState, new Values(options.globalKey));
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseQuery.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseQuery.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseQuery.java
index 4d95eb5..e3c8755 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseQuery.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseQuery.java
@@ -1,28 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.trident.state;
-import org.apache.storm.tuple.Values;
+import java.util.List;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseQueryFunction;
import org.apache.storm.trident.tuple.TridentTuple;
-
-import java.util.List;
+import org.apache.storm.tuple.Values;
public class HBaseQuery extends BaseQueryFunction<HBaseState, List<Values>> {
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseState.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseState.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseState.java
index 7b32eae..a216402 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseState.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseState.java
@@ -1,44 +1,41 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.trident.state;
-import org.apache.storm.Config;
-import org.apache.storm.topology.FailedException;
-import org.apache.storm.tuple.Values;
import com.google.common.collect.Lists;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.storm.Config;
import org.apache.storm.hbase.bolt.mapper.HBaseProjectionCriteria;
import org.apache.storm.hbase.bolt.mapper.HBaseValueMapper;
import org.apache.storm.hbase.common.ColumnList;
import org.apache.storm.hbase.common.HBaseClient;
import org.apache.storm.hbase.trident.mapper.TridentHBaseMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.storm.topology.FailedException;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.tuple.TridentTuple;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import org.apache.storm.tuple.Values;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HBaseState implements State {
@@ -57,49 +54,10 @@ public class HBaseState implements State {
this.numPartitions = numPartitions;
}
- public static class Options implements Serializable {
- private TridentHBaseMapper mapper;
- private Durability durability = Durability.SKIP_WAL;
- private HBaseProjectionCriteria projectionCriteria;
- private HBaseValueMapper rowToStormValueMapper;
- private String configKey;
- private String tableName;
-
- public Options withDurability(Durability durability) {
- this.durability = durability;
- return this;
- }
-
- public Options withProjectionCriteria(HBaseProjectionCriteria projectionCriteria) {
- this.projectionCriteria = projectionCriteria;
- return this;
- }
-
- public Options withConfigKey(String configKey) {
- this.configKey = configKey;
- return this;
- }
-
- public Options withTableName(String tableName) {
- this.tableName = tableName;
- return this;
- }
-
- public Options withRowToStormValueMapper(HBaseValueMapper rowToStormValueMapper) {
- this.rowToStormValueMapper = rowToStormValueMapper;
- return this;
- }
-
- public Options withMapper(TridentHBaseMapper mapper) {
- this.mapper = mapper;
- return this;
- }
- }
-
protected void prepare() {
final Configuration hbConfig = HBaseConfiguration.create();
Map<String, Object> conf = (Map<String, Object>) map.get(options.configKey);
- if(conf == null){
+ if (conf == null) {
LOG.info("HBase configuration not found using key '" + options.configKey + "'");
LOG.info("Using HBase config from first hbase-site.xml found on classpath.");
} else {
@@ -156,7 +114,7 @@ public class HBaseState implements State {
try {
Result[] results = hBaseClient.batchGet(gets);
- for(int i = 0; i < results.length; i++) {
+ for (int i = 0; i < results.length; i++) {
Result result = results[i];
TridentTuple tuple = tridentTuples.get(i);
List<Values> values = options.rowToStormValueMapper.toValues(tuple, result);
@@ -168,4 +126,43 @@ public class HBaseState implements State {
}
return batchRetrieveResult;
}
+
+ public static class Options implements Serializable {
+ private TridentHBaseMapper mapper;
+ private Durability durability = Durability.SKIP_WAL;
+ private HBaseProjectionCriteria projectionCriteria;
+ private HBaseValueMapper rowToStormValueMapper;
+ private String configKey;
+ private String tableName;
+
+ public Options withDurability(Durability durability) {
+ this.durability = durability;
+ return this;
+ }
+
+ public Options withProjectionCriteria(HBaseProjectionCriteria projectionCriteria) {
+ this.projectionCriteria = projectionCriteria;
+ return this;
+ }
+
+ public Options withConfigKey(String configKey) {
+ this.configKey = configKey;
+ return this;
+ }
+
+ public Options withTableName(String tableName) {
+ this.tableName = tableName;
+ return this;
+ }
+
+ public Options withRowToStormValueMapper(HBaseValueMapper rowToStormValueMapper) {
+ this.rowToStormValueMapper = rowToStormValueMapper;
+ return this;
+ }
+
+ public Options withMapper(TridentHBaseMapper mapper) {
+ this.mapper = mapper;
+ return this;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseStateFactory.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseStateFactory.java
index 08e9dc3..99b87b5 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseStateFactory.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseStateFactory.java
@@ -1,28 +1,22 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.trident.state;
+import java.util.Map;
import org.apache.storm.task.IMetricsContext;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.state.StateFactory;
-import java.util.Map;
-
public class HBaseStateFactory implements StateFactory {
private HBaseState.Options options;
@@ -33,7 +27,7 @@ public class HBaseStateFactory implements StateFactory {
@Override
public State makeState(Map<String, Object> map, IMetricsContext iMetricsContext, int partitionIndex, int numPartitions) {
- HBaseState state = new HBaseState(map , partitionIndex, numPartitions, options);
+ HBaseState state = new HBaseState(map, partitionIndex, numPartitions, options);
state.prepare();
return state;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseUpdater.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseUpdater.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseUpdater.java
index 57ca844..6340abc 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseUpdater.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/state/HBaseUpdater.java
@@ -1,29 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.trident.state;
+import java.util.List;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.state.BaseStateUpdater;
import org.apache.storm.trident.tuple.TridentTuple;
-import java.util.List;
-
-public class HBaseUpdater extends BaseStateUpdater<HBaseState> {
+public class HBaseUpdater extends BaseStateUpdater<HBaseState> {
@Override
public void updateState(HBaseState hBaseState, List<TridentTuple> tuples, TridentCollector collector) {
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/windowing/HBaseWindowsStore.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/windowing/HBaseWindowsStore.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/windowing/HBaseWindowsStore.java
index 702a790..086b477 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/windowing/HBaseWindowsStore.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/windowing/HBaseWindowsStore.java
@@ -1,23 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.trident.windowing;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
@@ -31,32 +35,20 @@ import org.apache.storm.trident.windowing.WindowsStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
/**
* This class stores entries into hbase instance of the given configuration.
- *
*/
public class HBaseWindowsStore implements WindowsStore {
- private static final Logger LOG = LoggerFactory.getLogger(HBaseWindowsStore.class);
public static final String UTF_8 = "utf-8";
-
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseWindowsStore.class);
private final ThreadLocal<HTable> threadLocalHtable;
private final ThreadLocal<WindowKryoSerializer> threadLocalWindowKryoSerializer;
private final Queue<HTable> htables = new ConcurrentLinkedQueue<>();
private final byte[] family;
private final byte[] qualifier;
- public HBaseWindowsStore(final Map<String, Object> topoConf, final Configuration config, final String tableName, byte[] family, byte[] qualifier) {
+ public HBaseWindowsStore(final Map<String, Object> topoConf, final Configuration config, final String tableName, byte[] family,
+ byte[] qualifier) {
this.family = family;
this.qualifier = qualifier;
@@ -73,7 +65,7 @@ public class HBaseWindowsStore implements WindowsStore {
}
};
- threadLocalWindowKryoSerializer = new ThreadLocal<WindowKryoSerializer>(){
+ threadLocalWindowKryoSerializer = new ThreadLocal<WindowKryoSerializer>() {
@Override
protected WindowKryoSerializer initialValue() {
return new WindowKryoSerializer(topoConf);
@@ -111,7 +103,7 @@ public class HBaseWindowsStore implements WindowsStore {
throw new RuntimeException(e);
}
- if(result.isEmpty()) {
+ if (result.isEmpty()) {
return null;
}
@@ -136,11 +128,11 @@ public class HBaseWindowsStore implements WindowsStore {
}
List<Object> values = new ArrayList<>();
- for (int i=0; i<results.length; i++) {
+ for (int i = 0; i < results.length; i++) {
Result result = results[i];
- if(result.isEmpty()) {
+ if (result.isEmpty()) {
LOG.error("Got empty result for key [{}]", keys.get(i));
- throw new RuntimeException("Received empty result for key: "+keys.get(i));
+ throw new RuntimeException("Received empty result for key: " + keys.get(i));
}
Object resultObject = windowKryoSerializer().deserialize(result.getValue(family, qualifier));
values.add(resultObject);
@@ -200,8 +192,8 @@ public class HBaseWindowsStore implements WindowsStore {
WindowsStore.Entry.nonNullCheckForKey(key);
WindowsStore.Entry.nonNullCheckForValue(value);
- if(value == null) {
- throw new IllegalArgumentException("Invalid value of null with key: "+key);
+ if (value == null) {
+ throw new IllegalArgumentException("Invalid value of null with key: " + key);
}
Put put = new Put(effectiveKey(key));
put.addColumn(family, ByteBuffer.wrap(qualifier), System.currentTimeMillis(), windowKryoSerializer().serializeToByteBuffer(value));
@@ -217,7 +209,8 @@ public class HBaseWindowsStore implements WindowsStore {
List<Put> list = new ArrayList<>();
for (Entry entry : entries) {
Put put = new Put(effectiveKey(entry.key));
- put.addColumn(family, ByteBuffer.wrap(qualifier), System.currentTimeMillis(), windowKryoSerializer().serializeToByteBuffer(entry.value));
+ put.addColumn(family, ByteBuffer.wrap(qualifier), System.currentTimeMillis(),
+ windowKryoSerializer().serializeToByteBuffer(entry.value));
list.add(put);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/windowing/HBaseWindowsStoreFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/windowing/HBaseWindowsStoreFactory.java b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/windowing/HBaseWindowsStoreFactory.java
index f0c8805..9981811 100644
--- a/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/windowing/HBaseWindowsStoreFactory.java
+++ b/external/storm-hbase/src/main/java/org/apache/storm/hbase/trident/windowing/HBaseWindowsStoreFactory.java
@@ -1,33 +1,25 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.hbase.trident.windowing;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.storm.trident.windowing.WindowsStore;
import org.apache.storm.trident.windowing.WindowsStoreFactory;
-import java.util.Map;
-
/**
* Factory to create {@link HBaseWindowsStore} instances.
- *
*/
public class HBaseWindowsStoreFactory implements WindowsStoreFactory {
private final Map<String, Object> config;
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseClientTestUtil.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseClientTestUtil.java b/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseClientTestUtil.java
index c4fa7c1..24ac8c5 100644
--- a/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseClientTestUtil.java
+++ b/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseClientTestUtil.java
@@ -18,11 +18,7 @@
package org.apache.storm.hbase.state;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-
import com.google.common.primitives.UnsignedBytes;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -34,7 +30,6 @@ import java.util.NavigableSet;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
-
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
@@ -52,27 +47,30 @@ import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+
public class HBaseClientTestUtil {
private HBaseClientTestUtil() {
}
public static HBaseClient mockedHBaseClient() throws Exception {
return mockedHBaseClient(new ConcurrentSkipListMap<byte[], NavigableMap<byte[], NavigableMap<byte[], byte[]>>>(
- UnsignedBytes.lexicographicalComparator()));
+ UnsignedBytes.lexicographicalComparator()));
}
public static HBaseClient mockedHBaseClient(
- ConcurrentNavigableMap<byte[], NavigableMap<byte[], NavigableMap<byte[], byte[]>>> internalMap)
- throws Exception {
+ ConcurrentNavigableMap<byte[], NavigableMap<byte[], NavigableMap<byte[], byte[]>>> internalMap)
+ throws Exception {
HBaseClient mockClient = mock(HBaseClient.class);
Mockito.doNothing().when(mockClient).close();
Mockito.when(mockClient.constructGetRequests(any(byte[].class), any(HBaseProjectionCriteria.class)))
- .thenCallRealMethod();
+ .thenCallRealMethod();
Mockito.when(mockClient.constructMutationReq(any(byte[].class), any(ColumnList.class), any(Durability.class)))
- .thenCallRealMethod();
+ .thenCallRealMethod();
Mockito.when(mockClient.exists(any(Get.class))).thenAnswer(new ExistsAnswer(internalMap));
Mockito.when(mockClient.batchGet(any(List.class))).thenAnswer(new BatchGetAnswer(internalMap));
@@ -84,8 +82,8 @@ public class HBaseClientTestUtil {
static class BuildCellsHelper {
public static void addMatchingColumnFamilies(byte[] rowKey, Map<byte[], NavigableSet<byte[]>> familyMap,
- NavigableMap<byte[], NavigableMap<byte[], byte[]>> cfToQualifierToValueMap,
- List<Cell> cells) {
+ NavigableMap<byte[], NavigableMap<byte[], byte[]>> cfToQualifierToValueMap,
+ List<Cell> cells) {
for (Map.Entry<byte[], NavigableSet<byte[]>> entry : familyMap.entrySet()) {
byte[] columnFamily = entry.getKey();
@@ -101,9 +99,9 @@ public class HBaseClientTestUtil {
}
public static void addMatchingQualifiers(byte[] rowKey, byte[] columnFamily,
- Map.Entry<byte[], NavigableSet<byte[]>> qualifierSet,
- NavigableMap<byte[], byte[]> qualifierToValueMap,
- List<Cell> cells) {
+ Map.Entry<byte[], NavigableSet<byte[]>> qualifierSet,
+ NavigableMap<byte[], byte[]> qualifierToValueMap,
+ List<Cell> cells) {
for (byte[] qualifier : qualifierSet.getValue()) {
byte[] value = qualifierToValueMap.get(qualifier);
if (value != null) {
@@ -113,7 +111,7 @@ public class HBaseClientTestUtil {
}
public static void addAllColumnFamilies(byte[] rowKey, NavigableMap<byte[], NavigableMap<byte[], byte[]>> cfToQualifierToValueMap,
- List<Cell> cells) {
+ List<Cell> cells) {
for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> entry : cfToQualifierToValueMap.entrySet()) {
byte[] columnFamily = entry.getKey();
addAllQualifiers(rowKey, columnFamily, entry.getValue(), cells);
@@ -121,7 +119,7 @@ public class HBaseClientTestUtil {
}
public static void addAllQualifiers(byte[] rowKey, byte[] columnFamily,
- NavigableMap<byte[], byte[]> qualifierToValueMap, List<Cell> cells) {
+ NavigableMap<byte[], byte[]> qualifierToValueMap, List<Cell> cells) {
for (Map.Entry<byte[], byte[]> entry2 : qualifierToValueMap.entrySet()) {
byte[] qualifier = entry2.getKey();
byte[] value = entry2.getValue();
@@ -149,7 +147,7 @@ public class HBaseClientTestUtil {
byte[] rowKey = get.getRow();
NavigableMap<byte[], NavigableMap<byte[], byte[]>> cfToQualifierToValueMap =
- mockMap.get(rowKey);
+ mockMap.get(rowKey);
if (cfToQualifierToValueMap != null) {
Map<byte[], NavigableSet<byte[]>> familyMap = get.getFamilyMap();
@@ -310,7 +308,7 @@ public class HBaseClientTestUtil {
byte[] endKey = (byte[]) args[1];
final ConcurrentNavigableMap<byte[], NavigableMap<byte[], NavigableMap<byte[], byte[]>>> subMap =
- mockMap.subMap(startKey, true, endKey, false);
+ mockMap.subMap(startKey, true, endKey, false);
final List<Result> results = buildResults(subMap);
@@ -353,7 +351,7 @@ public class HBaseClientTestUtil {
@Override
public Result[] next(int nbRows) throws IOException {
List<Result> bulkResult = new ArrayList<>();
- for (int i = 0 ; i < nbRows ; i++) {
+ for (int i = 0; i < nbRows; i++) {
Result result = next();
if (result == null) {
break;
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateIteratorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateIteratorTest.java b/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateIteratorTest.java
index aa965d1..250e337 100644
--- a/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateIteratorTest.java
+++ b/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateIteratorTest.java
@@ -16,13 +16,7 @@
package org.apache.storm.hbase.state;
-import static org.apache.storm.hbase.state.HBaseKeyValueState.STATE_QUALIFIER;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
import com.google.common.primitives.UnsignedBytes;
-
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -30,7 +24,6 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.TreeMap;
-
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Mutation;
@@ -42,6 +35,11 @@ import org.apache.storm.state.Serializer;
import org.junit.Before;
import org.junit.Test;
+import static org.apache.storm.hbase.state.HBaseKeyValueState.STATE_QUALIFIER;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
/**
* Test for HBaseKeyValueStateIterator.
*/
@@ -81,8 +79,8 @@ public class HBaseKeyValueStateIteratorTest {
applyPendingStateToHBase(chunkMap);
HBaseKeyValueStateIterator<byte[], byte[]> kvIterator =
- new HBaseKeyValueStateIterator<>(namespace, columnFamily, mockHBaseClient, pendingPrepare.entrySet().iterator(),
- pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
+ new HBaseKeyValueStateIterator<>(namespace, columnFamily, mockHBaseClient, pendingPrepare.entrySet().iterator(),
+ pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
assertNextEntry(kvIterator, "key0".getBytes(), "value0".getBytes());
@@ -111,8 +109,8 @@ public class HBaseKeyValueStateIteratorTest {
applyPendingStateToHBase(chunkMap);
HBaseKeyValueStateIterator<byte[], byte[]> kvIterator =
- new HBaseKeyValueStateIterator<>(namespace, columnFamily, mockHBaseClient, pendingPrepare.entrySet().iterator(),
- pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
+ new HBaseKeyValueStateIterator<>(namespace, columnFamily, mockHBaseClient, pendingPrepare.entrySet().iterator(),
+ pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
// keys shouldn't appear twice
@@ -134,8 +132,8 @@ public class HBaseKeyValueStateIteratorTest {
NavigableMap<byte[], byte[]> pendingCommit = getBinaryTreeMap();
HBaseKeyValueStateIterator<byte[], byte[]> kvIterator =
- new HBaseKeyValueStateIterator<>(namespace, columnFamily, mockHBaseClient, pendingPrepare.entrySet().iterator(),
- pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
+ new HBaseKeyValueStateIterator<>(namespace, columnFamily, mockHBaseClient, pendingPrepare.entrySet().iterator(),
+ pendingCommit.entrySet().iterator(), chunkSize, keySerializer, valueSerializer);
assertFalse(kvIterator.hasNext());
}
@@ -170,7 +168,7 @@ public class HBaseKeyValueStateIteratorTest {
mutations.add(new Delete(getRowKeyForStateKey(rowKey)));
} else {
List<Mutation> mutationsForRow = prepareMutateRow(getRowKeyForStateKey(rowKey), columnFamily,
- Collections.singletonMap(STATE_QUALIFIER, value));
+ Collections.singletonMap(STATE_QUALIFIER, value));
mutations.addAll(mutationsForRow);
}
}
@@ -204,7 +202,7 @@ public class HBaseKeyValueStateIteratorTest {
}
private void mutateRow(byte[] rowKey, byte[] columnFamily, Map<byte[], byte[]> map)
- throws Exception {
+ throws Exception {
mutateRow(rowKey, columnFamily, map, Durability.USE_DEFAULT);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateProviderTest.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateProviderTest.java b/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateProviderTest.java
index bd38e57..b7c562f 100644
--- a/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateProviderTest.java
+++ b/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateProviderTest.java
@@ -18,11 +18,10 @@
package org.apache.storm.hbase.state;
-import org.apache.storm.Config;
-import org.junit.Test;
-
import java.util.HashMap;
import java.util.Map;
+import org.apache.storm.Config;
+import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -38,7 +37,7 @@ public class HBaseKeyValueStateProviderTest {
HBaseKeyValueStateProvider provider = new HBaseKeyValueStateProvider();
Map<String, String> stormConf = new HashMap<>();
stormConf.put(Config.TOPOLOGY_STATE_PROVIDER_CONFIG, "{\"keyClass\":\"String\", \"valueClass\":\"String\"," +
- " \"tableName\": \"table\", \"columnFamily\": \"cf\"}");
+ " \"tableName\": \"table\", \"columnFamily\": \"cf\"}");
try {
HBaseKeyValueStateProvider.StateConfig config = provider.getStateConfig(stormConf);
@@ -53,7 +52,7 @@ public class HBaseKeyValueStateProviderTest {
HBaseKeyValueStateProvider provider = new HBaseKeyValueStateProvider();
Map<String, String> stormConf = new HashMap<>();
stormConf.put(Config.TOPOLOGY_STATE_PROVIDER_CONFIG, "{\"keyClass\":\"String\", \"valueClass\":\"String\"," +
- " \"hbaseConfigKey\": \"hbaseConfKey\", \"columnFamily\": \"cf\"}");
+ " \"hbaseConfigKey\": \"hbaseConfKey\", \"columnFamily\": \"cf\"}");
try {
HBaseKeyValueStateProvider.StateConfig config = provider.getStateConfig(stormConf);
@@ -68,7 +67,7 @@ public class HBaseKeyValueStateProviderTest {
HBaseKeyValueStateProvider provider = new HBaseKeyValueStateProvider();
Map<String, String> stormConf = new HashMap<>();
stormConf.put(Config.TOPOLOGY_STATE_PROVIDER_CONFIG, "{\"keyClass\":\"String\", \"valueClass\":\"String\"," +
- " \"hbaseConfigKey\": \"hbaseConfKey\", \"tableName\": \"table\"}");
+ " \"hbaseConfigKey\": \"hbaseConfKey\", \"tableName\": \"table\"}");
try {
HBaseKeyValueStateProvider.StateConfig config = provider.getStateConfig(stormConf);
@@ -83,8 +82,8 @@ public class HBaseKeyValueStateProviderTest {
HBaseKeyValueStateProvider provider = new HBaseKeyValueStateProvider();
Map<String, String> stormConf = new HashMap<>();
stormConf.put(Config.TOPOLOGY_STATE_PROVIDER_CONFIG, "{\"keyClass\":\"String\", \"valueClass\":\"String\"," +
- " \"hbaseConfigKey\": \"hbaseConfKey\", \"tableName\": \"table\"," +
- " \"columnFamily\": \"columnFamily\"}");
+ " \"hbaseConfigKey\": \"hbaseConfKey\", \"tableName\": \"table\"," +
+ " \"columnFamily\": \"columnFamily\"}");
HBaseKeyValueStateProvider.StateConfig config = provider.getStateConfig(stormConf);
assertEquals("String", config.keyClass);
http://git-wip-us.apache.org/repos/asf/storm/blob/880d14f1/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateTest.java
----------------------------------------------------------------------
diff --git a/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateTest.java b/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateTest.java
index 7827283..bf40409 100644
--- a/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateTest.java
+++ b/external/storm-hbase/src/test/java/org/apache/storm/hbase/state/HBaseKeyValueStateTest.java
@@ -19,17 +19,14 @@
package org.apache.storm.hbase.state;
import com.google.common.primitives.UnsignedBytes;
+import java.util.NavigableMap;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.storm.hbase.common.HBaseClient;
import org.apache.storm.state.DefaultStateSerializer;
import org.junit.Before;
import org.junit.Test;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentNavigableMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -49,7 +46,7 @@ public class HBaseKeyValueStateTest {
mockMap = new ConcurrentSkipListMap<>(UnsignedBytes.lexicographicalComparator());
mockClient = HBaseClientTestUtil.mockedHBaseClient(mockMap);
keyValueState = new HBaseKeyValueState<>(mockClient, COLUMN_FAMILY, NAMESPACE,
- new DefaultStateSerializer<String>(), new DefaultStateSerializer<String>());
+ new DefaultStateSerializer<String>(), new DefaultStateSerializer<String>());
}
@Test
@@ -80,38 +77,38 @@ public class HBaseKeyValueStateTest {
keyValueState.put("b", "2");
keyValueState.prepareCommit(1);
keyValueState.put("c", "3");
- assertArrayEquals(new String[]{"1", "2", "3"}, getValues());
+ assertArrayEquals(new String[]{ "1", "2", "3" }, getValues());
keyValueState.rollback();
- assertArrayEquals(new String[]{null, null, null}, getValues());
+ assertArrayEquals(new String[]{ null, null, null }, getValues());
keyValueState.put("a", "1");
keyValueState.put("b", "2");
keyValueState.prepareCommit(1);
keyValueState.commit(1);
keyValueState.put("c", "3");
- assertArrayEquals(new String[]{"1", "2", "3"}, getValues());
+ assertArrayEquals(new String[]{ "1", "2", "3" }, getValues());
keyValueState.rollback();
- assertArrayEquals(new String[]{"1", "2", null}, getValues());
+ assertArrayEquals(new String[]{ "1", "2", null }, getValues());
keyValueState.put("c", "3");
assertEquals("2", keyValueState.delete("b"));
assertEquals("3", keyValueState.delete("c"));
- assertArrayEquals(new String[]{"1", null, null}, getValues());
+ assertArrayEquals(new String[]{ "1", null, null }, getValues());
keyValueState.prepareCommit(2);
- assertArrayEquals(new String[]{"1", null, null}, getValues());
+ assertArrayEquals(new String[]{ "1", null, null }, getValues());
keyValueState.commit(2);
- assertArrayEquals(new String[]{"1", null, null}, getValues());
+ assertArrayEquals(new String[]{ "1", null, null }, getValues());
keyValueState.put("b", "2");
keyValueState.prepareCommit(3);
keyValueState.put("c", "3");
- assertArrayEquals(new String[]{"1", "2", "3"}, getValues());
+ assertArrayEquals(new String[]{ "1", "2", "3" }, getValues());
keyValueState.rollback();
- assertArrayEquals(new String[]{"1", null, null}, getValues());
+ assertArrayEquals(new String[]{ "1", null, null }, getValues());
}
private String[] getValues() {
return new String[]{
- keyValueState.get("a"),
- keyValueState.get("b"),
- keyValueState.get("c")
+ keyValueState.get("a"),
+ keyValueState.get("b"),
+ keyValueState.get("c")
};
}
}
\ No newline at end of file
[40/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-starter
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java
index fafdbcc..9db1dd6 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountAggBolt.java
@@ -1,23 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.bolt;
-import org.apache.storm.Config;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.log4j.Logger;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -25,52 +22,45 @@ import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
-import org.apache.log4j.Logger;
-import org.apache.storm.starter.tools.NthLastModifiedTimeTracker;
-import org.apache.storm.starter.tools.SlidingWindowCounter;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
/**
* This bolt aggregates counts from multiple upstream bolts.
*/
public class RollingCountAggBolt extends BaseRichBolt {
- private static final long serialVersionUID = 5537727428628598519L;
- private static final Logger LOG = Logger.getLogger(RollingCountAggBolt.class);
- //Mapping of key->upstreamBolt->count
- private Map<Object, Map<Integer, Long>> counts = new HashMap<Object, Map<Integer, Long>>();
- private OutputCollector collector;
-
+ private static final long serialVersionUID = 5537727428628598519L;
+ private static final Logger LOG = Logger.getLogger(RollingCountAggBolt.class);
+ //Mapping of key->upstreamBolt->count
+ private Map<Object, Map<Integer, Long>> counts = new HashMap<Object, Map<Integer, Long>>();
+ private OutputCollector collector;
- @Override
- public void prepare(Map<String, Object> topoConf, TopologyContext context, OutputCollector collector) {
- this.collector = collector;
- }
- @Override
- public void execute(Tuple tuple) {
- Object obj = tuple.getValue(0);
- long count = tuple.getLong(1);
- int source = tuple.getSourceTask();
- Map<Integer, Long> subCounts = counts.get(obj);
- if (subCounts == null) {
- subCounts = new HashMap<Integer, Long>();
- counts.put(obj, subCounts);
+ @Override
+ public void prepare(Map<String, Object> topoConf, TopologyContext context, OutputCollector collector) {
+ this.collector = collector;
}
- //Update the current count for this object
- subCounts.put(source, count);
- //Output the sum of all the known counts so for this key
- long sum = 0;
- for (Long val: subCounts.values()) {
- sum += val;
+
+ @Override
+ public void execute(Tuple tuple) {
+ Object obj = tuple.getValue(0);
+ long count = tuple.getLong(1);
+ int source = tuple.getSourceTask();
+ Map<Integer, Long> subCounts = counts.get(obj);
+ if (subCounts == null) {
+ subCounts = new HashMap<Integer, Long>();
+ counts.put(obj, subCounts);
+ }
+ //Update the current count for this object
+ subCounts.put(source, count);
+ //Output the sum of all the known counts so for this key
+ long sum = 0;
+ for (Long val : subCounts.values()) {
+ sum += val;
+ }
+ collector.emit(new Values(obj, sum));
}
- collector.emit(new Values(obj, sum));
- }
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("obj", "count"));
- }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("obj", "count"));
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java
index b7ea55e..0e5cef5 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/RollingCountBolt.java
@@ -1,23 +1,24 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.bolt;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import org.apache.log4j.Logger;
import org.apache.storm.Config;
+import org.apache.storm.starter.tools.NthLastModifiedTimeTracker;
+import org.apache.storm.starter.tools.SlidingWindowCounter;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -26,13 +27,6 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.TupleUtils;
-import org.apache.log4j.Logger;
-import org.apache.storm.starter.tools.NthLastModifiedTimeTracker;
-import org.apache.storm.starter.tools.SlidingWindowCounter;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
/**
* This bolt performs rolling counts of incoming objects, i.e. sliding window based counting.
@@ -55,87 +49,86 @@ import java.util.Map.Entry;
*/
public class RollingCountBolt extends BaseRichBolt {
- private static final long serialVersionUID = 5537727428628598519L;
- private static final Logger LOG = Logger.getLogger(RollingCountBolt.class);
- private static final int NUM_WINDOW_CHUNKS = 5;
- private static final int DEFAULT_SLIDING_WINDOW_IN_SECONDS = NUM_WINDOW_CHUNKS * 60;
- private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = DEFAULT_SLIDING_WINDOW_IN_SECONDS / NUM_WINDOW_CHUNKS;
- private static final String WINDOW_LENGTH_WARNING_TEMPLATE =
- "Actual window length is %d seconds when it should be %d seconds"
- + " (you can safely ignore this warning during the startup phase)";
+ private static final long serialVersionUID = 5537727428628598519L;
+ private static final Logger LOG = Logger.getLogger(RollingCountBolt.class);
+ private static final int NUM_WINDOW_CHUNKS = 5;
+ private static final int DEFAULT_SLIDING_WINDOW_IN_SECONDS = NUM_WINDOW_CHUNKS * 60;
+ private static final int DEFAULT_EMIT_FREQUENCY_IN_SECONDS = DEFAULT_SLIDING_WINDOW_IN_SECONDS / NUM_WINDOW_CHUNKS;
+ private static final String WINDOW_LENGTH_WARNING_TEMPLATE =
+ "Actual window length is %d seconds when it should be %d seconds"
+ + " (you can safely ignore this warning during the startup phase)";
- private final SlidingWindowCounter<Object> counter;
- private final int windowLengthInSeconds;
- private final int emitFrequencyInSeconds;
- private OutputCollector collector;
- private NthLastModifiedTimeTracker lastModifiedTracker;
+ private final SlidingWindowCounter<Object> counter;
+ private final int windowLengthInSeconds;
+ private final int emitFrequencyInSeconds;
+ private OutputCollector collector;
+ private NthLastModifiedTimeTracker lastModifiedTracker;
- public RollingCountBolt() {
- this(DEFAULT_SLIDING_WINDOW_IN_SECONDS, DEFAULT_EMIT_FREQUENCY_IN_SECONDS);
- }
-
- public RollingCountBolt(int windowLengthInSeconds, int emitFrequencyInSeconds) {
- this.windowLengthInSeconds = windowLengthInSeconds;
- this.emitFrequencyInSeconds = emitFrequencyInSeconds;
- counter = new SlidingWindowCounter<Object>(deriveNumWindowChunksFrom(this.windowLengthInSeconds,
- this.emitFrequencyInSeconds));
- }
+ public RollingCountBolt() {
+ this(DEFAULT_SLIDING_WINDOW_IN_SECONDS, DEFAULT_EMIT_FREQUENCY_IN_SECONDS);
+ }
- private int deriveNumWindowChunksFrom(int windowLengthInSeconds, int windowUpdateFrequencyInSeconds) {
- return windowLengthInSeconds / windowUpdateFrequencyInSeconds;
- }
+ public RollingCountBolt(int windowLengthInSeconds, int emitFrequencyInSeconds) {
+ this.windowLengthInSeconds = windowLengthInSeconds;
+ this.emitFrequencyInSeconds = emitFrequencyInSeconds;
+ counter = new SlidingWindowCounter<Object>(deriveNumWindowChunksFrom(this.windowLengthInSeconds,
+ this.emitFrequencyInSeconds));
+ }
- @Override
- public void prepare(Map<String, Object> topoConf, TopologyContext context, OutputCollector collector) {
- this.collector = collector;
- lastModifiedTracker = new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds,
- this.emitFrequencyInSeconds));
- }
+ private int deriveNumWindowChunksFrom(int windowLengthInSeconds, int windowUpdateFrequencyInSeconds) {
+ return windowLengthInSeconds / windowUpdateFrequencyInSeconds;
+ }
- @Override
- public void execute(Tuple tuple) {
- if (TupleUtils.isTick(tuple)) {
- LOG.debug("Received tick tuple, triggering emit of current window counts");
- emitCurrentWindowCounts();
+ @Override
+ public void prepare(Map<String, Object> topoConf, TopologyContext context, OutputCollector collector) {
+ this.collector = collector;
+ lastModifiedTracker = new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds,
+ this.emitFrequencyInSeconds));
}
- else {
- countObjAndAck(tuple);
+
+ @Override
+ public void execute(Tuple tuple) {
+ if (TupleUtils.isTick(tuple)) {
+ LOG.debug("Received tick tuple, triggering emit of current window counts");
+ emitCurrentWindowCounts();
+ } else {
+ countObjAndAck(tuple);
+ }
}
- }
- private void emitCurrentWindowCounts() {
- Map<Object, Long> counts = counter.getCountsThenAdvanceWindow();
- int actualWindowLengthInSeconds = lastModifiedTracker.secondsSinceOldestModification();
- lastModifiedTracker.markAsModified();
- if (actualWindowLengthInSeconds != windowLengthInSeconds) {
- LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds, windowLengthInSeconds));
+ private void emitCurrentWindowCounts() {
+ Map<Object, Long> counts = counter.getCountsThenAdvanceWindow();
+ int actualWindowLengthInSeconds = lastModifiedTracker.secondsSinceOldestModification();
+ lastModifiedTracker.markAsModified();
+ if (actualWindowLengthInSeconds != windowLengthInSeconds) {
+ LOG.warn(String.format(WINDOW_LENGTH_WARNING_TEMPLATE, actualWindowLengthInSeconds, windowLengthInSeconds));
+ }
+ emit(counts, actualWindowLengthInSeconds);
}
- emit(counts, actualWindowLengthInSeconds);
- }
- private void emit(Map<Object, Long> counts, int actualWindowLengthInSeconds) {
- for (Entry<Object, Long> entry : counts.entrySet()) {
- Object obj = entry.getKey();
- Long count = entry.getValue();
- collector.emit(new Values(obj, count, actualWindowLengthInSeconds));
+ private void emit(Map<Object, Long> counts, int actualWindowLengthInSeconds) {
+ for (Entry<Object, Long> entry : counts.entrySet()) {
+ Object obj = entry.getKey();
+ Long count = entry.getValue();
+ collector.emit(new Values(obj, count, actualWindowLengthInSeconds));
+ }
}
- }
- private void countObjAndAck(Tuple tuple) {
- Object obj = tuple.getValue(0);
- counter.incrementCount(obj);
- collector.ack(tuple);
- }
+ private void countObjAndAck(Tuple tuple) {
+ Object obj = tuple.getValue(0);
+ counter.incrementCount(obj);
+ collector.ack(tuple);
+ }
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("obj", "count", "actualWindowLengthInSeconds"));
- }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("obj", "count", "actualWindowLengthInSeconds"));
+ }
- @Override
- public Map<String, Object> getComponentConfiguration() {
- Map<String, Object> conf = new HashMap<String, Object>();
- conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, emitFrequencyInSeconds);
- return conf;
- }
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ Map<String, Object> conf = new HashMap<String, Object>();
+ conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, emitFrequencyInSeconds);
+ return conf;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java
index aaed340..00a2ab7 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SingleJoinBolt.java
@@ -1,22 +1,23 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.bolt;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.storm.Config;
import org.apache.storm.generated.GlobalStreamId;
import org.apache.storm.task.OutputCollector;
@@ -27,92 +28,92 @@ import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.utils.TimeCacheMap;
-import java.util.*;
-
/** Example of a simple custom bolt for joining two streams
* NOTE: Prefer to use the built-in JoinBolt wherever applicable
*/
public class SingleJoinBolt extends BaseRichBolt {
- OutputCollector _collector;
- Fields _idFields;
- Fields _outFields;
- int _numSources;
- TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>> _pending;
- Map<String, GlobalStreamId> _fieldLocations;
+ OutputCollector _collector;
+ Fields _idFields;
+ Fields _outFields;
+ int _numSources;
+ TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>> _pending;
+ Map<String, GlobalStreamId> _fieldLocations;
- public SingleJoinBolt(Fields outFields) {
- _outFields = outFields;
- }
+ public SingleJoinBolt(Fields outFields) {
+ _outFields = outFields;
+ }
- @Override
- public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
- _fieldLocations = new HashMap<String, GlobalStreamId>();
- _collector = collector;
- int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue();
- _pending = new TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>>(timeout, new ExpireCallback());
- _numSources = context.getThisSources().size();
- Set<String> idFields = null;
- for (GlobalStreamId source : context.getThisSources().keySet()) {
- Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId());
- Set<String> setFields = new HashSet<String>(fields.toList());
- if (idFields == null)
- idFields = setFields;
- else
- idFields.retainAll(setFields);
+ @Override
+ public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
+ _fieldLocations = new HashMap<String, GlobalStreamId>();
+ _collector = collector;
+ int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue();
+ _pending = new TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>>(timeout, new ExpireCallback());
+ _numSources = context.getThisSources().size();
+ Set<String> idFields = null;
+ for (GlobalStreamId source : context.getThisSources().keySet()) {
+ Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId());
+ Set<String> setFields = new HashSet<String>(fields.toList());
+ if (idFields == null) {
+ idFields = setFields;
+ } else {
+ idFields.retainAll(setFields);
+ }
- for (String outfield : _outFields) {
- for (String sourcefield : fields) {
- if (outfield.equals(sourcefield)) {
- _fieldLocations.put(outfield, source);
- }
+ for (String outfield : _outFields) {
+ for (String sourcefield : fields) {
+ if (outfield.equals(sourcefield)) {
+ _fieldLocations.put(outfield, source);
+ }
+ }
+ }
}
- }
- }
- _idFields = new Fields(new ArrayList<String>(idFields));
+ _idFields = new Fields(new ArrayList<String>(idFields));
- if (_fieldLocations.size() != _outFields.size()) {
- throw new RuntimeException("Cannot find all outfields among sources");
+ if (_fieldLocations.size() != _outFields.size()) {
+ throw new RuntimeException("Cannot find all outfields among sources");
+ }
}
- }
- @Override
- public void execute(Tuple tuple) {
- List<Object> id = tuple.select(_idFields);
- GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId());
- if (!_pending.containsKey(id)) {
- _pending.put(id, new HashMap<GlobalStreamId, Tuple>());
- }
- Map<GlobalStreamId, Tuple> parts = _pending.get(id);
- if (parts.containsKey(streamId))
- throw new RuntimeException("Received same side of single join twice");
- parts.put(streamId, tuple);
- if (parts.size() == _numSources) {
- _pending.remove(id);
- List<Object> joinResult = new ArrayList<Object>();
- for (String outField : _outFields) {
- GlobalStreamId loc = _fieldLocations.get(outField);
- joinResult.add(parts.get(loc).getValueByField(outField));
- }
- _collector.emit(new ArrayList<Tuple>(parts.values()), joinResult);
+ @Override
+ public void execute(Tuple tuple) {
+ List<Object> id = tuple.select(_idFields);
+ GlobalStreamId streamId = new GlobalStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId());
+ if (!_pending.containsKey(id)) {
+ _pending.put(id, new HashMap<GlobalStreamId, Tuple>());
+ }
+ Map<GlobalStreamId, Tuple> parts = _pending.get(id);
+ if (parts.containsKey(streamId)) {
+ throw new RuntimeException("Received same side of single join twice");
+ }
+ parts.put(streamId, tuple);
+ if (parts.size() == _numSources) {
+ _pending.remove(id);
+ List<Object> joinResult = new ArrayList<Object>();
+ for (String outField : _outFields) {
+ GlobalStreamId loc = _fieldLocations.get(outField);
+ joinResult.add(parts.get(loc).getValueByField(outField));
+ }
+ _collector.emit(new ArrayList<Tuple>(parts.values()), joinResult);
- for (Tuple part : parts.values()) {
- _collector.ack(part);
- }
+ for (Tuple part : parts.values()) {
+ _collector.ack(part);
+ }
+ }
}
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(_outFields);
- }
- private class ExpireCallback implements TimeCacheMap.ExpiredCallback<List<Object>, Map<GlobalStreamId, Tuple>> {
@Override
- public void expire(List<Object> id, Map<GlobalStreamId, Tuple> tuples) {
- for (Tuple tuple : tuples.values()) {
- _collector.fail(tuple);
- }
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(_outFields);
+ }
+
+ private class ExpireCallback implements TimeCacheMap.ExpiredCallback<List<Object>, Map<GlobalStreamId, Tuple>> {
+ @Override
+ public void expire(List<Object> id, Map<GlobalStreamId, Tuple> tuples) {
+ for (Tuple tuple : tuples.values()) {
+ _collector.fail(tuple);
+ }
+ }
}
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java
index cf9801f..7987717 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/SlidingWindowSumBolt.java
@@ -1,22 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.bolt;
+import java.util.List;
+import java.util.Map;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -28,9 +25,6 @@ import org.apache.storm.windowing.TupleWindow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.List;
-import java.util.Map;
-
/**
* Computes sliding window sum
*/
@@ -47,23 +41,23 @@ public class SlidingWindowSumBolt extends BaseWindowedBolt {
@Override
public void execute(TupleWindow inputWindow) {
- /*
- * The inputWindow gives a view of
- * (a) all the events in the window
- * (b) events that expired since last activation of the window
- * (c) events that newly arrived since last activation of the window
- */
+ /*
+ * The inputWindow gives a view of
+ * (a) all the events in the window
+ * (b) events that expired since last activation of the window
+ * (c) events that newly arrived since last activation of the window
+ */
List<Tuple> tuplesInWindow = inputWindow.get();
List<Tuple> newTuples = inputWindow.getNew();
List<Tuple> expiredTuples = inputWindow.getExpired();
LOG.debug("Events in current window: " + tuplesInWindow.size());
- /*
- * Instead of iterating over all the tuples in the window to compute
- * the sum, the values for the new events are added and old events are
- * subtracted. Similar optimizations might be possible in other
- * windowing computations.
- */
+ /*
+ * Instead of iterating over all the tuples in the window to compute
+ * the sum, the values for the new events are added and old events are
+ * subtracted. Similar optimizations might be possible in other
+ * windowing computations.
+ */
for (Tuple tuple : newTuples) {
sum += (int) tuple.getValue(0);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java
index bfed34e..e185a9e 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/bolt/TotalRankingsBolt.java
@@ -1,25 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.bolt;
-import org.apache.storm.tuple.Tuple;
import org.apache.log4j.Logger;
import org.apache.storm.starter.tools.Rankings;
+import org.apache.storm.tuple.Tuple;
/**
* This bolt merges incoming {@link Rankings}.
@@ -29,31 +24,31 @@ import org.apache.storm.starter.tools.Rankings;
*/
public final class TotalRankingsBolt extends AbstractRankerBolt {
- private static final long serialVersionUID = -8447525895532302198L;
- private static final Logger LOG = Logger.getLogger(TotalRankingsBolt.class);
+ private static final long serialVersionUID = -8447525895532302198L;
+ private static final Logger LOG = Logger.getLogger(TotalRankingsBolt.class);
- public TotalRankingsBolt() {
- super();
- }
+ public TotalRankingsBolt() {
+ super();
+ }
- public TotalRankingsBolt(int topN) {
- super(topN);
- }
+ public TotalRankingsBolt(int topN) {
+ super(topN);
+ }
- public TotalRankingsBolt(int topN, int emitFrequencyInSeconds) {
- super(topN, emitFrequencyInSeconds);
- }
+ public TotalRankingsBolt(int topN, int emitFrequencyInSeconds) {
+ super(topN, emitFrequencyInSeconds);
+ }
- @Override
- void updateRankingsWithTuple(Tuple tuple) {
- Rankings rankingsToBeMerged = (Rankings) tuple.getValue(0);
- super.getRankings().updateWith(rankingsToBeMerged);
- super.getRankings().pruneZeroCounts();
- }
+ @Override
+ void updateRankingsWithTuple(Tuple tuple) {
+ Rankings rankingsToBeMerged = (Rankings) tuple.getValue(0);
+ super.getRankings().updateWith(rankingsToBeMerged);
+ super.getRankings().pruneZeroCounts();
+ }
- @Override
- Logger getLogger() {
- return LOG;
- }
+ @Override
+ Logger getLogger() {
+ return LOG;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java
index 37b2ec4..e1a9ee5 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomIntegerSpout.java
@@ -1,22 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.spout;
+import java.util.Map;
+import java.util.Random;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -27,9 +24,6 @@ import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
-import java.util.Random;
-
/**
* Emits a random integer and a timestamp value (offset by one day),
* every 100 ms. The ts field can be used in tuple time based windowing.
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java
index 866cc80..e6c4804 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomNumberGeneratorSpout.java
@@ -1,34 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-package org.apache.storm.starter.spout;
-import org.apache.storm.Config;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.trident.operation.TridentCollector;
-import org.apache.storm.trident.spout.IBatchSpout;
-import org.apache.storm.tuple.Fields;
+package org.apache.storm.starter.spout;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
+import org.apache.storm.Config;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.spout.IBatchSpout;
+import org.apache.storm.tuple.Fields;
/**
* This spout generates random whole numbers with given {@code maxNumber} value as maximum with the given {@code fields}.
@@ -53,13 +46,13 @@ public class RandomNumberGeneratorSpout implements IBatchSpout {
@Override
public void emitBatch(long batchId, TridentCollector collector) {
List<List<Object>> values = null;
- if(batches.containsKey(batchId)) {
+ if (batches.containsKey(batchId)) {
values = batches.get(batchId);
} else {
values = new ArrayList<>();
for (int i = 0; i < batchSize; i++) {
List<Object> numbers = new ArrayList<>();
- for (int x=0; x<fields.size(); x++) {
+ for (int x = 0; x < fields.size(); x++) {
numbers.add(ThreadLocalRandom.current().nextInt(0, maxNumber + 1));
}
values.add(numbers);
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java
index 733c9d9..e3e0a68 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/spout/RandomSentenceSpout.java
@@ -1,22 +1,21 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.spout;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Map;
+import java.util.Random;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -27,71 +26,68 @@ import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.Map;
-import java.util.Random;
-
public class RandomSentenceSpout extends BaseRichSpout {
- private static final Logger LOG = LoggerFactory.getLogger(RandomSentenceSpout.class);
+ private static final Logger LOG = LoggerFactory.getLogger(RandomSentenceSpout.class);
- SpoutOutputCollector _collector;
- Random _rand;
+ SpoutOutputCollector _collector;
+ Random _rand;
- @Override
- public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
- _collector = collector;
- _rand = new Random();
- }
+ @Override
+ public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
+ _collector = collector;
+ _rand = new Random();
+ }
- @Override
- public void nextTuple() {
- Utils.sleep(100);
- String[] sentences = new String[]{sentence("the cow jumped over the moon"), sentence("an apple a day keeps the doctor away"),
- sentence("four score and seven years ago"), sentence("snow white and the seven dwarfs"), sentence("i am at two with nature")};
- final String sentence = sentences[_rand.nextInt(sentences.length)];
+ @Override
+ public void nextTuple() {
+ Utils.sleep(100);
+ String[] sentences = new String[]{
+ sentence("the cow jumped over the moon"), sentence("an apple a day keeps the doctor away"),
+ sentence("four score and seven years ago"), sentence("snow white and the seven dwarfs"), sentence("i am at two with nature")
+ };
+ final String sentence = sentences[_rand.nextInt(sentences.length)];
- LOG.debug("Emitting tuple: {}", sentence);
+ LOG.debug("Emitting tuple: {}", sentence);
- _collector.emit(new Values(sentence));
- }
+ _collector.emit(new Values(sentence));
+ }
- protected String sentence(String input) {
- return input;
- }
+ protected String sentence(String input) {
+ return input;
+ }
- @Override
- public void ack(Object id) {
- }
+ @Override
+ public void ack(Object id) {
+ }
- @Override
- public void fail(Object id) {
- }
+ @Override
+ public void fail(Object id) {
+ }
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("word"));
- }
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("word"));
+ }
- // Add unique identifier to each tuple, which is helpful for debugging
- public static class TimeStamped extends RandomSentenceSpout {
- private final String prefix;
+ // Add unique identifier to each tuple, which is helpful for debugging
+ public static class TimeStamped extends RandomSentenceSpout {
+ private final String prefix;
- public TimeStamped() {
- this("");
- }
+ public TimeStamped() {
+ this("");
+ }
- public TimeStamped(String prefix) {
- this.prefix = prefix;
- }
+ public TimeStamped(String prefix) {
+ this.prefix = prefix;
+ }
- protected String sentence(String input) {
- return prefix + currentDate() + " " + input;
- }
+ protected String sentence(String input) {
+ return prefix + currentDate() + " " + input;
+ }
- private String currentDate() {
- return new SimpleDateFormat("yyyy.MM.dd_HH:mm:ss.SSSSSSSSS").format(new Date());
+ private String currentDate() {
+ return new SimpleDateFormat("yyyy.MM.dd_HH:mm:ss.SSSSSSSSS").format(new Date());
+ }
}
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java
index dae774b..feaff4f 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/AggregateExample.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.streams;
import org.apache.storm.Config;
@@ -40,10 +35,10 @@ public class AggregateExample {
* results are merged and the final result is emitted.
*/
builder.newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0), 2)
- .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(5)))
- .filter(x -> x > 0 && x < 500)
- .aggregate(new Avg())
- .print();
+ .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(5)))
+ .filter(x -> x > 0 && x < 500)
+ .aggregate(new Avg())
+ .print();
Config config = new Config();
String topoName = "AGG_EXAMPLE";
@@ -69,14 +64,14 @@ public class AggregateExample {
public Pair<Integer, Integer> merge(Pair<Integer, Integer> sumAndCount1, Pair<Integer, Integer> sumAndCount2) {
System.out.println("Merge " + sumAndCount1 + " and " + sumAndCount2);
return Pair.of(
- sumAndCount1._1 + sumAndCount2._1,
- sumAndCount1._2 + sumAndCount2._2
+ sumAndCount1._1 + sumAndCount2._1,
+ sumAndCount1._2 + sumAndCount2._2
);
}
@Override
public Double result(Pair<Integer, Integer> sumAndCount) {
- return (double) sumAndCount._1/sumAndCount._2;
+ return (double) sumAndCount._1 / sumAndCount._2;
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java
index f8d865f..d20f071 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/BranchExample.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.streams;
import org.apache.storm.Config;
@@ -38,18 +33,18 @@ public class BranchExample {
public static void main(String[] args) throws Exception {
StreamBuilder builder = new StreamBuilder();
Stream<Integer>[] evenAndOdd = builder
- /*
- * Create a stream of random numbers from a spout that
- * emits random integers by extracting the tuple value at index 0.
- */
- .newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0))
- /*
- * Split the stream of numbers into streams of
- * even and odd numbers. The first stream contains even
- * and the second contains odd numbers.
- */
- .branch(x -> (x % 2) == 0,
- x -> (x % 2) == 1);
+ /*
+ * Create a stream of random numbers from a spout that
+ * emits random integers by extracting the tuple value at index 0.
+ */
+ .newStream(new RandomIntegerSpout(), new ValueMapper<Integer>(0))
+ /*
+ * Split the stream of numbers into streams of
+ * even and odd numbers. The first stream contains even
+ * and the second contains odd numbers.
+ */
+ .branch(x -> (x % 2) == 0,
+ x -> (x % 2) == 1);
evenAndOdd[0].forEach(x -> LOG.info("EVEN> " + x));
evenAndOdd[1].forEach(x -> LOG.info("ODD > " + x));
@@ -59,7 +54,7 @@ public class BranchExample {
if (args.length > 0) {
topoName = args[0];
}
-
+
config.setNumWorkers(1);
StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java
index 1e154b7..6bcf194 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/GroupByKeyAndWindowExample.java
@@ -1,26 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.streams;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.spout.SpoutOutputCollector;
@@ -48,27 +42,27 @@ public class GroupByKeyAndWindowExample {
// a stream of stock quotes
builder.newStream(new StockQuotes(), new PairValueMapper<String, Double>(0, 1))
- /*
- * The elements having the same key within the window will be grouped
- * together and the corresponding values will be merged.
- *
- * The result is a PairStream<String, Iterable<Double>> with
- * 'stock symbol' as the key and 'stock prices' for that symbol within the window as the value.
- */
- .groupByKeyAndWindow(SlidingWindows.of(Count.of(6), Count.of(3)))
- .print();
+ /*
+ * The elements having the same key within the window will be grouped
+ * together and the corresponding values will be merged.
+ *
+ * The result is a PairStream<String, Iterable<Double>> with
+ * 'stock symbol' as the key and 'stock prices' for that symbol within the window as the value.
+ */
+ .groupByKeyAndWindow(SlidingWindows.of(Count.of(6), Count.of(3)))
+ .print();
// a stream of stock quotes
builder.newStream(new StockQuotes(), new PairValueMapper<String, Double>(0, 1))
- /*
- * The elements having the same key within the window will be grouped
- * together and their values will be reduced using the given reduce function.
- *
- * Here the result is a PairStream<String, Double> with
- * 'stock symbol' as the key and the maximum price for that symbol within the window as the value.
- */
- .reduceByKeyAndWindow((x, y) -> x > y ? x : y, SlidingWindows.of(Count.of(6), Count.of(3)))
- .print();
+ /*
+ * The elements having the same key within the window will be grouped
+ * together and their values will be reduced using the given reduce function.
+ *
+ * Here the result is a PairStream<String, Double> with
+ * 'stock symbol' as the key and the maximum price for that symbol within the window as the value.
+ */
+ .reduceByKeyAndWindow((x, y) -> x > y ? x : y, SlidingWindows.of(Count.of(6), Count.of(3)))
+ .print();
Config config = new Config();
String topoName = GroupByKeyAndWindowExample.class.getName();
@@ -81,9 +75,9 @@ public class GroupByKeyAndWindowExample {
private static class StockQuotes extends BaseRichSpout {
private final List<List<Values>> values = Arrays.asList(
- Arrays.asList(new Values("AAPL", 100.0), new Values("GOOG", 780.0), new Values("FB", 125.0)),
- Arrays.asList(new Values("AAPL", 105.0), new Values("GOOG", 790.0), new Values("FB", 130.0)),
- Arrays.asList(new Values("AAPL", 102.0), new Values("GOOG", 788.0), new Values("FB", 128.0))
+ Arrays.asList(new Values("AAPL", 100.0), new Values("GOOG", 780.0), new Values("FB", 125.0)),
+ Arrays.asList(new Values("AAPL", 105.0), new Values("GOOG", 790.0), new Values("FB", 130.0)),
+ Arrays.asList(new Values("AAPL", 102.0), new Values("GOOG", 788.0), new Values("FB", 128.0))
);
private SpoutOutputCollector collector;
private int index = 0;
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java
index 020239d..52bbf28 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/JoinExample.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.streams;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.spout.SpoutOutputCollector;
@@ -44,25 +38,25 @@ public class JoinExample {
StreamBuilder builder = new StreamBuilder();
// a stream of (number, square) pairs
PairStream<Integer, Integer> squares = builder
- .newStream(new NumberSpout(x -> x * x),
- new PairValueMapper<>(0, 1));
+ .newStream(new NumberSpout(x -> x * x),
+ new PairValueMapper<>(0, 1));
// a stream of (number, cube) pairs
PairStream<Integer, Integer> cubes = builder
- .newStream(new NumberSpout(x -> x * x * x),
- new PairValueMapper<>(0, 1));
+ .newStream(new NumberSpout(x -> x * x * x),
+ new PairValueMapper<>(0, 1));
// create a windowed stream of five seconds duration
squares.window(TumblingWindows.of(Duration.seconds(5)))
- /*
- * Join the squares and the cubes stream within the window.
- * The values in the squares stream having the same key as that
- * of the cubes stream within the window will be joined together.
- */
- .join(cubes)
- /**
- * The results should be of the form (number, (square, cube))
- */
- .print();
+ /*
+ * Join the squares and the cubes stream within the window.
+ * The values in the squares stream having the same key as that
+ * of the cubes stream within the window will be joined together.
+ */
+ .join(cubes)
+ /**
+ * The results should be of the form (number, (square, cube))
+ */
+ .print();
Config config = new Config();
String topoName = JoinExample.class.getName();
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java
index 3cd1199..4e5b1ef 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StateQueryExample.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.streams;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.spout.SpoutOutputCollector;
@@ -53,27 +47,28 @@ public class StateQueryExample {
public static void main(String[] args) throws Exception {
StreamBuilder builder = new StreamBuilder();
StreamState<String, Long> ss = builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2)
- /*
- * Transform the stream of words to a stream of (word, 1) pairs
- */
- .mapToPair(w -> Pair.of(w, 1))
- /*
- * Update the count in the state. Here the first argument 0L is the initial value for the count and
- * the second argument is a function that increments the count for each value received.
- */
- .updateStateByKey(0L, (count, val) -> count + 1);
+ /*
+ * Transform the stream of words to a stream of (word, 1) pairs
+ */
+ .mapToPair(w -> Pair.of(w, 1))
+ /*
+ * Update the count in the state. Here the first argument 0L is the initial value for the
+ * count and
+ * the second argument is a function that increments the count for each value received.
+ */
+ .updateStateByKey(0L, (count, val) -> count + 1);
/*
* A stream of words emitted by the QuerySpout is used as
* the keys to query the state.
*/
builder.newStream(new QuerySpout(), new ValueMapper<String>(0))
- /*
- * Queries the state and emits the
- * matching (key, value) as results. The stream state returned
- * by the updateStateByKey is passed as the argument to stateQuery.
- */
- .stateQuery(ss).print();
+ /*
+ * Queries the state and emits the
+ * matching (key, value) as results. The stream state returned
+ * by the updateStateByKey is passed as the argument to stateQuery.
+ */
+ .stateQuery(ss).print();
Config config = new Config();
// use redis based state store for persistence
@@ -87,8 +82,8 @@ public class StateQueryExample {
}
private static class QuerySpout extends BaseRichSpout {
+ private final String[] words = { "nathan", "mike" };
private SpoutOutputCollector collector;
- private final String[] words = {"nathan", "mike"};
@Override
public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java
index aba19f3..afc345d 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/StatefulWordCount.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.streams;
import org.apache.storm.Config;
@@ -48,26 +43,26 @@ public class StatefulWordCount {
StreamBuilder builder = new StreamBuilder();
// a stream of words
builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2)
- .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(2)))
- /*
- * create a stream of (word, 1) pairs
- */
- .mapToPair(w -> Pair.of(w, 1))
- /*
- * compute the word counts in the last two second window
- */
- .countByKey()
- /*
- * update the word counts in the state.
- * Here the first argument 0L is the initial value for the state
- * and the second argument is a function that adds the count to the current value in the state.
- */
- .updateStateByKey(0L, (state, count) -> state + count)
- /*
- * convert the state back to a stream and print the results
- */
- .toPairStream()
- .print();
+ .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(2)))
+ /*
+ * create a stream of (word, 1) pairs
+ */
+ .mapToPair(w -> Pair.of(w, 1))
+ /*
+ * compute the word counts in the last two second window
+ */
+ .countByKey()
+ /*
+ * update the word counts in the state.
+ * Here the first argument 0L is the initial value for the state
+ * and the second argument is a function that adds the count to the current value in the state.
+ */
+ .updateStateByKey(0L, (state, count) -> state + count)
+ /*
+ * convert the state back to a stream and print the results
+ */
+ .toPairStream()
+ .print();
Config config = new Config();
// use redis based state store for persistence
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java
index a74151b..e859ca1 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/TypedTupleExample.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.streams;
import org.apache.storm.Config;
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java
index 50c8aad..2c06674 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WindowedWordCount.java
@@ -1,24 +1,18 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.streams;
import java.util.Arrays;
-
import org.apache.storm.Config;
import org.apache.storm.StormSubmitter;
import org.apache.storm.starter.spout.RandomSentenceSpout;
@@ -36,31 +30,31 @@ public class WindowedWordCount {
StreamBuilder builder = new StreamBuilder();
// A stream of random sentences
builder.newStream(new RandomSentenceSpout(), new ValueMapper<String>(0), 2)
- /*
- * a two seconds tumbling window
- */
- .window(TumblingWindows.of(Duration.seconds(2)))
- /*
- * split the sentences to words
- */
- .flatMap(s -> Arrays.asList(s.split(" ")))
- /*
- * create a stream of (word, 1) pairs
- */
- .mapToPair(w -> Pair.of(w, 1))
- /*
- * compute the word counts in the last two second window
- */
- .countByKey()
- /*
- * emit the count for the words that occurred
- * at-least five times in the last two seconds
- */
- .filter(x -> x.getSecond() >= 5)
- /*
- * print the results to stdout
- */
- .print();
+ /*
+ * a two seconds tumbling window
+ */
+ .window(TumblingWindows.of(Duration.seconds(2)))
+ /*
+ * split the sentences to words
+ */
+ .flatMap(s -> Arrays.asList(s.split(" ")))
+ /*
+ * create a stream of (word, 1) pairs
+ */
+ .mapToPair(w -> Pair.of(w, 1))
+ /*
+ * compute the word counts in the last two second window
+ */
+ .countByKey()
+ /*
+ * emit the count for the words that occurred
+ * at-least five times in the last two seconds
+ */
+ .filter(x -> x.getSecond() >= 5)
+ /*
+ * print the results to stdout
+ */
+ .print();
Config config = new Config();
String topoName = "test";
http://git-wip-us.apache.org/repos/asf/storm/blob/81ec15d1/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java
----------------------------------------------------------------------
diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java
index 360f0ad..082b96a 100644
--- a/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java
+++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/streams/WordCountToBolt.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.starter.streams;
import org.apache.storm.Config;
@@ -40,7 +35,7 @@ public class WordCountToBolt {
// Redis config parameters for the RedisStoreBolt
JedisPoolConfig poolConfig = new JedisPoolConfig.Builder()
- .setHost("127.0.0.1").setPort(6379).build();
+ .setHost("127.0.0.1").setPort(6379).build();
// Storm tuple to redis key-value mapper
RedisStoreMapper storeMapper = new WordCountStoreMapper();
// The redis bolt (sink)
@@ -48,21 +43,21 @@ public class WordCountToBolt {
// A stream of words
builder.newStream(new TestWordSpout(), new ValueMapper<String>(0))
- /*
- * create a stream of (word, 1) pairs
- */
- .mapToPair(w -> Pair.of(w, 1))
- /*
- * aggregate the count
- */
- .countByKey()
- /*
- * The result of aggregation is forwarded to
- * the RedisStoreBolt. The forwarded tuple is a
- * key-value pair of (word, count) with ("key", "value")
- * being the field names.
- */
- .to(redisStoreBolt);
+ /*
+ * create a stream of (word, 1) pairs
+ */
+ .mapToPair(w -> Pair.of(w, 1))
+ /*
+ * aggregate the count
+ */
+ .countByKey()
+ /*
+ * The result of aggregation is forwarded to
+ * the RedisStoreBolt. The forwarded tuple is a
+ * key-value pair of (word, count) with ("key", "value")
+ * being the field names.
+ */
+ .to(redisStoreBolt);
Config config = new Config();
String topoName = "test";
[45/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-kafka
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/TridentKafkaTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/TridentKafkaTest.java b/external/storm-kafka/src/test/org/apache/storm/kafka/TridentKafkaTest.java
index 864eaa9..926b5fe 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/TridentKafkaTest.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/TridentKafkaTest.java
@@ -1,27 +1,20 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
-import org.apache.storm.tuple.Fields;
+import java.util.ArrayList;
+import java.util.List;
import kafka.javaapi.consumer.SimpleConsumer;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
import org.apache.storm.kafka.trident.TridentKafkaState;
import org.apache.storm.kafka.trident.mapper.FieldNameBasedTupleToKafkaMapper;
import org.apache.storm.kafka.trident.mapper.TridentTupleToKafkaMapper;
@@ -29,9 +22,10 @@ import org.apache.storm.kafka.trident.selector.DefaultTopicSelector;
import org.apache.storm.kafka.trident.selector.KafkaTopicSelector;
import org.apache.storm.trident.tuple.TridentTuple;
import org.apache.storm.trident.tuple.TridentTupleView;
-
-import java.util.ArrayList;
-import java.util.List;
+import org.apache.storm.tuple.Fields;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
public class TridentKafkaTest {
private KafkaTestBroker broker;
@@ -45,8 +39,8 @@ public class TridentKafkaTest {
TridentTupleToKafkaMapper<Object, Object> mapper = new FieldNameBasedTupleToKafkaMapper<Object, Object>("key", "message");
KafkaTopicSelector topicSelector = new DefaultTopicSelector(TestUtils.TOPIC);
state = new TridentKafkaState()
- .withKafkaTopicSelector(topicSelector)
- .withTridentTupleToKafkaMapper(mapper);
+ .withKafkaTopicSelector(topicSelector)
+ .withTridentTupleToKafkaMapper(mapper);
state.prepare(TestUtils.getProducerProperties(broker.getBrokerConnectionString()));
}
@@ -60,14 +54,14 @@ public class TridentKafkaTest {
state.updateState(tridentTuples, null);
- for(int i = 0 ; i < batchSize ; i++) {
+ for (int i = 0; i < batchSize; i++) {
TestUtils.verifyMessage(keyString, valString, broker, simpleConsumer);
}
}
private List<TridentTuple> generateTupleBatch(String key, String message, int batchsize) {
List<TridentTuple> batch = new ArrayList<>();
- for(int i =0 ; i < batchsize; i++) {
+ for (int i = 0; i < batchsize; i++) {
batch.add(TridentTupleView.createFreshTuple(new Fields("key", "message"), key, message));
}
return batch;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/ZkCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/ZkCoordinatorTest.java b/external/storm-kafka/src/test/org/apache/storm/kafka/ZkCoordinatorTest.java
index 31dfffe..93709cf 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/ZkCoordinatorTest.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/ZkCoordinatorTest.java
@@ -1,34 +1,38 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka;
-import org.apache.storm.Config;
-import org.apache.curator.test.TestingServer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import kafka.javaapi.consumer.SimpleConsumer;
+import org.apache.curator.test.TestingServer;
+import org.apache.storm.Config;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
-import java.util.*;
-
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.when;
@@ -60,7 +64,7 @@ public class ZkCoordinatorTest {
Map<String, Object> conf = buildZookeeperConfig(server);
state = new ZkState(conf);
simpleConsumer = new SimpleConsumer("localhost", broker.getPort(), 60000, 1024, "testClient");
- when(dynamicPartitionConnections.register(any(Broker.class), any(String.class) ,anyInt())).thenReturn(simpleConsumer);
+ when(dynamicPartitionConnections.register(any(Broker.class), any(String.class), anyInt())).thenReturn(simpleConsumer);
}
private Map<String, Object> buildZookeeperConfig(TestingServer server) {
@@ -128,7 +132,8 @@ public class ZkCoordinatorTest {
HashMap<Integer, PartitionManager> managersAfterRefresh = new HashMap<Integer, PartitionManager>();
for (List<PartitionManager> partitionManagersAfter : partitionManagersAfterRefresh) {
for (PartitionManager manager : partitionManagersAfter) {
- assertFalse("Multiple PartitionManagers for same partition", managersAfterRefresh.containsKey(manager.getPartition().partition));
+ assertFalse("Multiple PartitionManagers for same partition",
+ managersAfterRefresh.containsKey(manager.getPartition().partition));
managersAfterRefresh.put(manager.getPartition().partition, manager);
}
}
@@ -150,7 +155,8 @@ public class ZkCoordinatorTest {
assertSame(managerBefore._committedTo, managerAfter._committedTo);
}
- private void assertPartitionsAreDifferent(List<PartitionManager> partitionManagersBefore, List<PartitionManager> partitionManagersAfter, int partitionsPerTask) {
+ private void assertPartitionsAreDifferent(List<PartitionManager> partitionManagersBefore, List<PartitionManager> partitionManagersAfter,
+ int partitionsPerTask) {
assertEquals(partitionsPerTask, partitionManagersBefore.size());
assertEquals(partitionManagersBefore.size(), partitionManagersAfter.size());
for (int i = 0; i < partitionsPerTask; i++) {
@@ -174,7 +180,8 @@ public class ZkCoordinatorTest {
private List<ZkCoordinator> buildCoordinators(int totalTasks) {
List<ZkCoordinator> coordinatorList = new ArrayList<ZkCoordinator>();
for (int i = 0; i < totalTasks; i++) {
- ZkCoordinator coordinator = new ZkCoordinator(dynamicPartitionConnections, topoConf, spoutConfig, state, i, totalTasks, i, "test-id", reader);
+ ZkCoordinator coordinator =
+ new ZkCoordinator(dynamicPartitionConnections, topoConf, spoutConfig, state, i, totalTasks, i, "test-id", reader);
coordinatorList.add(coordinator);
}
return coordinatorList;
http://git-wip-us.apache.org/repos/asf/storm/blob/4fe4f04b/external/storm-kafka/src/test/org/apache/storm/kafka/bolt/KafkaBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/test/org/apache/storm/kafka/bolt/KafkaBoltTest.java b/external/storm-kafka/src/test/org/apache/storm/kafka/bolt/KafkaBoltTest.java
index ed26157..5c3053c 100644
--- a/external/storm-kafka/src/test/org/apache/storm/kafka/bolt/KafkaBoltTest.java
+++ b/external/storm-kafka/src/test/org/apache/storm/kafka/bolt/KafkaBoltTest.java
@@ -1,37 +1,26 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.kafka.bolt;
-import org.apache.storm.Config;
-import org.apache.storm.Constants;
-import org.apache.storm.task.GeneralTopologyContext;
-import org.apache.storm.task.IOutputCollector;
-import org.apache.storm.task.OutputCollector;
-import org.apache.storm.topology.TopologyBuilder;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.tuple.Tuple;
-import org.apache.storm.tuple.TupleImpl;
-import org.apache.storm.tuple.Values;
-import org.apache.storm.utils.Utils;
-import org.apache.storm.utils.TupleUtils;
import com.google.common.collect.ImmutableList;
-import kafka.api.OffsetRequest;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.Future;
import kafka.api.FetchRequest;
+import kafka.api.OffsetRequest;
import kafka.javaapi.FetchResponse;
import kafka.javaapi.OffsetResponse;
import kafka.javaapi.consumer.SimpleConsumer;
@@ -41,25 +30,44 @@ import kafka.message.MessageAndOffset;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
-import org.junit.*;
+import org.apache.storm.Config;
+import org.apache.storm.Constants;
+import org.apache.storm.kafka.Broker;
+import org.apache.storm.kafka.BrokerHosts;
+import org.apache.storm.kafka.KafkaConfig;
+import org.apache.storm.kafka.KafkaTestBroker;
+import org.apache.storm.kafka.KafkaUtils;
+import org.apache.storm.kafka.Partition;
+import org.apache.storm.kafka.StaticHosts;
+import org.apache.storm.kafka.trident.GlobalPartitionInformation;
+import org.apache.storm.task.GeneralTopologyContext;
+import org.apache.storm.task.IOutputCollector;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.topology.TopologyBuilder;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.TupleImpl;
+import org.apache.storm.tuple.Values;
+import org.apache.storm.utils.TupleUtils;
+import org.apache.storm.utils.Utils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-import org.apache.storm.kafka.*;
-import org.apache.storm.kafka.trident.GlobalPartitionInformation;
-
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.HashMap;
-import java.util.Properties;
-import java.util.concurrent.Future;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.*;
-
-import java.lang.reflect.Field;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class KafkaBoltTest {
@@ -73,6 +81,29 @@ public class KafkaBoltTest {
@Mock
private IOutputCollector collector;
+ private static ByteBufferMessageSet mockSingleMessage(byte[] key, byte[] message) {
+ ByteBufferMessageSet sets = mock(ByteBufferMessageSet.class);
+ MessageAndOffset msg = mock(MessageAndOffset.class);
+ final List<MessageAndOffset> msgs = ImmutableList.of(msg);
+ doReturn(msgs.iterator()).when(sets).iterator();
+ Message kafkaMessage = mock(Message.class);
+ doReturn(ByteBuffer.wrap(key)).when(kafkaMessage).key();
+ doReturn(ByteBuffer.wrap(message)).when(kafkaMessage).payload();
+ doReturn(kafkaMessage).when(msg).message();
+ return sets;
+ }
+
+ private static SimpleConsumer mockSimpleConsumer(ByteBufferMessageSet mockMsg) {
+ SimpleConsumer simpleConsumer = mock(SimpleConsumer.class);
+ FetchResponse resp = mock(FetchResponse.class);
+ doReturn(resp).when(simpleConsumer).fetch(any(FetchRequest.class));
+ OffsetResponse mockOffsetResponse = mock(OffsetResponse.class);
+ doReturn(new long[]{}).when(mockOffsetResponse).offsets(anyString(), anyInt());
+ doReturn(mockOffsetResponse).when(simpleConsumer).getOffsetsBefore(any(kafka.javaapi.OffsetRequest.class));
+ doReturn(mockMsg).when(resp).messageSet(anyString(), anyInt());
+ return simpleConsumer;
+ }
+
@Before
public void initMocks() {
MockitoAnnotations.initMocks(this);
@@ -261,7 +292,6 @@ public class KafkaBoltTest {
verifyMessage(null, message);
}
-
@Test
public void executeWithBrokerDown() throws Exception {
broker.shutdown();
@@ -274,7 +304,9 @@ public class KafkaBoltTest {
private boolean verifyMessage(String key, String message) {
long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, kafkaConfig.topic, 0, OffsetRequest.LatestTime()) - 1;
ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(kafkaConfig, simpleConsumer,
- new Partition(Broker.fromString(broker.getBrokerConnectionString()),kafkaConfig.topic, 0), lastMessageOffset);
+ new Partition(
+ Broker.fromString(broker.getBrokerConnectionString()),
+ kafkaConfig.topic, 0), lastMessageOffset);
MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
Message kafkaMessage = messageAndOffset.message();
ByteBuffer messageKeyBuffer = kafkaMessage.key();
@@ -290,23 +322,25 @@ public class KafkaBoltTest {
private Tuple generateTestTuple(Object key, Object message) {
TopologyBuilder builder = new TopologyBuilder();
- GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") {
- @Override
- public Fields getComponentOutputFields(String componentId, String streamId) {
- return new Fields("key", "message");
- }
- };
+ GeneralTopologyContext topologyContext =
+ new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") {
+ @Override
+ public Fields getComponentOutputFields(String componentId, String streamId) {
+ return new Fields("key", "message");
+ }
+ };
return new TupleImpl(topologyContext, new Values(key, message), topologyContext.getComponentId(1), 1, "");
}
private Tuple generateTestTuple(Object message) {
TopologyBuilder builder = new TopologyBuilder();
- GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") {
- @Override
- public Fields getComponentOutputFields(String componentId, String streamId) {
- return new Fields("message");
- }
- };
+ GeneralTopologyContext topologyContext =
+ new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") {
+ @Override
+ public Fields getComponentOutputFields(String componentId, String streamId) {
+ return new Fields("message");
+ }
+ };
return new TupleImpl(topologyContext, new Values(message), topologyContext.getComponentId(1), 1, "");
}
@@ -318,27 +352,4 @@ public class KafkaBoltTest {
assertTrue(TupleUtils.isTick(tuple));
return tuple;
}
-
- private static ByteBufferMessageSet mockSingleMessage(byte[] key, byte[] message) {
- ByteBufferMessageSet sets = mock(ByteBufferMessageSet.class);
- MessageAndOffset msg = mock(MessageAndOffset.class);
- final List<MessageAndOffset> msgs = ImmutableList.of(msg);
- doReturn(msgs.iterator()).when(sets).iterator();
- Message kafkaMessage = mock(Message.class);
- doReturn(ByteBuffer.wrap(key)).when(kafkaMessage).key();
- doReturn(ByteBuffer.wrap(message)).when(kafkaMessage).payload();
- doReturn(kafkaMessage).when(msg).message();
- return sets;
- }
-
- private static SimpleConsumer mockSimpleConsumer(ByteBufferMessageSet mockMsg) {
- SimpleConsumer simpleConsumer = mock(SimpleConsumer.class);
- FetchResponse resp = mock(FetchResponse.class);
- doReturn(resp).when(simpleConsumer).fetch(any(FetchRequest.class));
- OffsetResponse mockOffsetResponse = mock(OffsetResponse.class);
- doReturn(new long[] {}).when(mockOffsetResponse).offsets(anyString(), anyInt());
- doReturn(mockOffsetResponse).when(simpleConsumer).getOffsetsBefore(any(kafka.javaapi.OffsetRequest.class));
- doReturn(mockMsg).when(resp).messageSet(anyString(), anyInt());
- return simpleConsumer;
- }
}
[15/63] [abbrv] storm git commit: Fixing stylecheck problemswith
storm-server
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Container.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Container.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Container.java
index 10d81e2..b4d6442 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Container.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Container.java
@@ -8,7 +8,7 @@
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -33,26 +33,23 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
import org.apache.storm.container.ResourceIsolationInterface;
import org.apache.storm.generated.LSWorkerHeartbeat;
import org.apache.storm.generated.LocalAssignment;
import org.apache.storm.generated.ProfileRequest;
-import org.apache.storm.generated.WorkerMetricPoint;
import org.apache.storm.generated.WorkerMetricList;
+import org.apache.storm.generated.WorkerMetricPoint;
import org.apache.storm.generated.WorkerMetrics;
import org.apache.storm.metric.StormMetricsRegistry;
import org.apache.storm.metricstore.MetricException;
import org.apache.storm.metricstore.WorkerMetricsProcessor;
import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.LocalState;
-import org.apache.storm.utils.NimbusClient;
import org.apache.storm.utils.ServerConfigUtils;
import org.apache.storm.utils.ServerUtils;
import org.apache.storm.utils.Utils;
-import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
@@ -66,50 +63,6 @@ public abstract class Container implements Killable {
private static final String SYSTEM_COMPONENT_ID = "System";
private static final String INVALID_EXECUTOR_ID = "-1";
private static final String INVALID_STREAM_ID = "None";
-
- public static enum ContainerType {
- LAUNCH(false, false),
- RECOVER_FULL(true, false),
- RECOVER_PARTIAL(true, true);
-
- private final boolean _recovery;
- private final boolean _onlyKillable;
-
- ContainerType(boolean recovery, boolean onlyKillable) {
- _recovery = recovery;
- _onlyKillable = onlyKillable;
- }
-
- public boolean isRecovery() {
- return _recovery;
- }
-
- public void assertFull() {
- if (_onlyKillable) {
- throw new IllegalStateException("Container is only Killable.");
- }
- }
-
- public boolean isOnlyKillable() {
- return _onlyKillable;
- }
- }
-
- private static class TopoAndMemory {
- public final String topoId;
- public final long memory;
-
- public TopoAndMemory(String id, long mem) {
- topoId = id;
- memory = mem;
- }
-
- @Override
- public String toString() {
- return "{TOPO: " + topoId + " at " + memory + " MB}";
- }
- }
-
private static final ConcurrentHashMap<Integer, TopoAndMemory> _usedMemory =
new ConcurrentHashMap<>();
private static final ConcurrentHashMap<Integer, TopoAndMemory> _reservedMemory =
@@ -139,10 +92,9 @@ public abstract class Container implements Killable {
return ret;
});
}
-
+
protected final Map<String, Object> _conf;
protected final Map<String, Object> _topoConf; //Not set if RECOVER_PARTIAL
- protected String _workerId;
protected final String _topologyId; //Not set if RECOVER_PARTIAL
protected final String _supervisorId;
protected final int _supervisorPort;
@@ -150,38 +102,39 @@ public abstract class Container implements Killable {
protected final LocalAssignment _assignment; //Not set if RECOVER_PARTIAL
protected final AdvancedFSOps _ops;
protected final ResourceIsolationInterface _resourceIsolationManager;
- protected ContainerType _type;
protected final boolean _symlinksDisabled;
+ protected String _workerId;
+ protected ContainerType _type;
private long lastMetricProcessTime = 0L;
-
/**
* Create a new Container.
- * @param type the type of container being made.
- * @param conf the supervisor config
- * @param supervisorId the ID of the supervisor this is a part of.
- * @param supervisorPort the thrift server port of the supervisor this is a part of.
- * @param port the port the container is on. Should be <= 0 if only a partial recovery
- * @param assignment the assignment for this container. Should be null if only a partial recovery.
+ *
+ * @param type the type of container being made.
+ * @param conf the supervisor config
+ * @param supervisorId the ID of the supervisor this is a part of.
+ * @param supervisorPort the thrift server port of the supervisor this is a part of.
+ * @param port the port the container is on. Should be <= 0 if only a partial recovery
+ * @param assignment the assignment for this container. Should be null if only a partial recovery.
* @param resourceIsolationManager used to isolate resources for a container can be null if no isolation is used.
- * @param workerId the id of the worker to use. Must not be null if doing a partial recovery.
- * @param topoConf the config of the topology (mostly for testing) if null
- * and not a partial recovery the real conf is read.
- * @param ops file system operations (mostly for testing) if null a new one is made
+ * @param workerId the id of the worker to use. Must not be null if doing a partial recovery.
+ * @param topoConf the config of the topology (mostly for testing) if null and not a partial recovery the real conf is
+ * read.
+ * @param ops file system operations (mostly for testing) if null a new one is made
* @throws IOException on any error.
*/
protected Container(ContainerType type, Map<String, Object> conf, String supervisorId, int supervisorPort,
- int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager,
- String workerId, Map<String, Object> topoConf, AdvancedFSOps ops) throws IOException {
- assert(type != null);
- assert(conf != null);
- assert(supervisorId != null);
-
- _symlinksDisabled = (boolean)conf.getOrDefault(Config.DISABLE_SYMLINKS, false);
-
+ int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager,
+ String workerId, Map<String, Object> topoConf, AdvancedFSOps ops) throws IOException {
+ assert (type != null);
+ assert (conf != null);
+ assert (supervisorId != null);
+
+ _symlinksDisabled = (boolean) conf.getOrDefault(Config.DISABLE_SYMLINKS, false);
+
if (ops == null) {
ops = AdvancedFSOps.make(conf);
}
-
+
_workerId = workerId;
_type = type;
_port = port;
@@ -191,20 +144,22 @@ public abstract class Container implements Killable {
_supervisorPort = supervisorPort;
_resourceIsolationManager = resourceIsolationManager;
_assignment = assignment;
-
+
if (_type.isOnlyKillable()) {
- assert(_assignment == null);
- assert(_port <= 0);
- assert(_workerId != null);
+ assert (_assignment == null);
+ assert (_port <= 0);
+ assert (_workerId != null);
_topologyId = null;
_topoConf = null;
} else {
- assert(assignment != null);
- assert(port > 0);
+ assert (assignment != null);
+ assert (port > 0);
_topologyId = assignment.get_topology_id();
if (!_ops.doRequiredTopoFilesExist(_conf, _topologyId)) {
- LOG.info("Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}", _assignment,
- _supervisorId, _port, _workerId);
+ LOG.info(
+ "Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}",
+ _assignment,
+ _supervisorId, _port, _workerId);
throw new ContainerRecoveryException("Missing required topology files...");
}
if (topoConf == null) {
@@ -215,35 +170,37 @@ public abstract class Container implements Killable {
}
}
}
-
+
@Override
public String toString() {
return "topo:" + _topologyId + " worker:" + _workerId;
}
-
+
protected Map<String, Object> readTopoConf() throws IOException {
- assert(_topologyId != null);
+ assert (_topologyId != null);
return ConfigUtils.readSupervisorStormConf(_conf, _topologyId);
}
-
+
/**
* Kill a given process.
+ *
* @param pid the id of the process to kill
* @throws IOException
*/
protected void kill(long pid) throws IOException {
ServerUtils.killProcessWithSigTerm(String.valueOf(pid));
}
-
+
/**
* Kill a given process.
+ *
* @param pid the id of the process to kill
* @throws IOException
*/
protected void forceKill(long pid) throws IOException {
ServerUtils.forceKillProcess(String.valueOf(pid));
}
-
+
@Override
public void kill() throws IOException {
LOG.info("Killing {}:{}", _supervisorId, _workerId);
@@ -253,20 +210,22 @@ public abstract class Container implements Killable {
kill(pid);
}
}
-
+
@Override
public void forceKill() throws IOException {
LOG.info("Force Killing {}:{}", _supervisorId, _workerId);
Set<Long> pids = getAllPids();
-
+
for (Long pid : pids) {
forceKill(pid);
}
}
-
+
/**
* Read the Heartbeat for the current container.
+ *
* @return the Heartbeat
+ *
* @throws IOException on any error
*/
public LSWorkerHeartbeat readHeartbeat() throws IOException {
@@ -278,9 +237,11 @@ public abstract class Container implements Killable {
/**
* Is a process alive and running?.
- * @param pid the PID of the running process
+ *
+ * @param pid the PID of the running process
* @param user the user that is expected to own that process
* @return true if it is, else false
+ *
* @throws IOException on any error
*/
protected boolean isProcessAlive(long pid, String user) throws IOException {
@@ -289,7 +250,7 @@ public abstract class Container implements Killable {
}
return isPosixProcessAlive(pid, user);
}
-
+
private boolean isWindowsProcessAlive(long pid, String user) throws IOException {
boolean ret = false;
ProcessBuilder pb = new ProcessBuilder("tasklist", "/fo", "list", "/fi", "pid eq " + pid, "/v");
@@ -302,16 +263,17 @@ public abstract class Container implements Killable {
//This line contains the user name for the pid we're looking up
//Example line: "User Name: exampleDomain\exampleUser"
List<String> userNameLineSplitOnWhitespace = Arrays.asList(read.split(":"));
- if(userNameLineSplitOnWhitespace.size() == 2){
+ if (userNameLineSplitOnWhitespace.size() == 2) {
List<String> userAndMaybeDomain = Arrays.asList(userNameLineSplitOnWhitespace.get(1).trim().split("\\\\"));
String processUser = userAndMaybeDomain.size() == 2 ? userAndMaybeDomain.get(1) : userAndMaybeDomain.get(0);
- if(user.equals(processUser)){
+ if (user.equals(processUser)) {
ret = true;
} else {
LOG.info("Found {} running as {}, but expected it to be {}", pid, processUser, user);
}
} else {
- LOG.error("Received unexpected output from tasklist command. Expected one colon in user name line. Line was {}", read);
+ LOG.error("Received unexpected output from tasklist command. Expected one colon in user name line. Line was {}",
+ read);
}
break;
}
@@ -319,7 +281,7 @@ public abstract class Container implements Killable {
}
return ret;
}
-
+
private boolean isPosixProcessAlive(long pid, String user) throws IOException {
boolean ret = false;
ProcessBuilder pb = new ProcessBuilder("ps", "-o", "user", "-p", String.valueOf(pid));
@@ -327,7 +289,7 @@ public abstract class Container implements Killable {
Process p = pb.start();
try (BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
String first = in.readLine();
- assert("USER".equals(first));
+ assert ("USER".equals(first));
String processUser;
while ((processUser = in.readLine()) != null) {
if (user.equals(processUser)) {
@@ -340,14 +302,14 @@ public abstract class Container implements Killable {
}
return ret;
}
-
+
@Override
public boolean areAllProcessesDead() throws IOException {
Set<Long> pids = getAllPids();
String user = getWorkerUser();
-
+
boolean allDead = true;
- for (Long pid: pids) {
+ for (Long pid : pids) {
if (!isProcessAlive(pid, user)) {
LOG.debug("{}: PID {} is dead", _workerId, pid);
} else {
@@ -364,42 +326,43 @@ public abstract class Container implements Killable {
_reservedMemory.remove(_port);
cleanUpForRestart();
}
-
+
/**
- * Setup the container to run. By default this creates the needed directories/links in the
- * local file system
- * PREREQUISITE: All needed blobs and topology, jars/configs have been downloaded and
- * placed in the appropriate locations
+ * Setup the container to run. By default this creates the needed directories/links in the local file system PREREQUISITE: All needed
+ * blobs and topology, jars/configs have been downloaded and placed in the appropriate locations
+ *
* @throws IOException on any error
*/
protected void setup() throws IOException {
_type.assertFull();
if (!_ops.doRequiredTopoFilesExist(_conf, _topologyId)) {
- LOG.info("Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}", _assignment,
- _supervisorId, _port, _workerId);
+ LOG.info("Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}",
+ _assignment,
+ _supervisorId, _port, _workerId);
throw new IllegalStateException("Not all needed files are here!!!!");
- }
+ }
LOG.info("Setting up {}:{}", _supervisorId, _workerId);
_ops.forceMkdir(new File(ConfigUtils.workerPidsRoot(_conf, _workerId)));
_ops.forceMkdir(new File(ConfigUtils.workerTmpRoot(_conf, _workerId)));
_ops.forceMkdir(new File(ConfigUtils.workerHeartbeatsRoot(_conf, _workerId)));
-
+
File workerArtifacts = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port));
if (!_ops.fileExists(workerArtifacts)) {
_ops.forceMkdir(workerArtifacts);
_ops.setupWorkerArtifactsDir(_assignment.get_owner(), workerArtifacts);
}
-
+
String user = getWorkerUser();
writeLogMetadata(user);
saveWorkerUser(user);
createArtifactsLink();
createBlobstoreLinks();
}
-
+
/**
* Write out the file used by the log viewer to allow/reject log access.
+ *
* @param user the user this is going to run as
* @throws IOException on any error
*/
@@ -413,7 +376,7 @@ public abstract class Container implements Killable {
Set<String> logsGroups = new HashSet<>();
if (_topoConf.get(DaemonConfig.LOGS_GROUPS) != null) {
List<String> groups = (List<String>) _topoConf.get(DaemonConfig.LOGS_GROUPS);
- for (String group : groups){
+ for (String group : groups) {
logsGroups.add(group);
}
}
@@ -426,13 +389,13 @@ public abstract class Container implements Killable {
Set<String> logsUsers = new HashSet<>();
if (_topoConf.get(DaemonConfig.LOGS_USERS) != null) {
List<String> logUsers = (List<String>) _topoConf.get(DaemonConfig.LOGS_USERS);
- for (String logUser : logUsers){
+ for (String logUser : logUsers) {
logsUsers.add(logUser);
}
}
if (_topoConf.get(Config.TOPOLOGY_USERS) != null) {
List<String> topUsers = (List<String>) _topoConf.get(Config.TOPOLOGY_USERS);
- for (String logUser : topUsers){
+ for (String logUser : topUsers) {
logsUsers.add(logUser);
}
}
@@ -445,9 +408,10 @@ public abstract class Container implements Killable {
yaml.dump(data, writer);
}
}
-
+
/**
* Create symlink from the containers directory/artifacts to the artifacts directory.
+ *
* @throws IOException on any error
*/
protected void createArtifactsLink() throws IOException {
@@ -461,17 +425,17 @@ public abstract class Container implements Killable {
}
}
}
-
+
/**
- * Create symlinks for each of the blobs from the container's directory to
- * corresponding links in the storm dist directory.
+ * Create symlinks for each of the blobs from the container's directory to corresponding links in the storm dist directory.
+ *
* @throws IOException on any error.
*/
protected void createBlobstoreLinks() throws IOException {
_type.assertFull();
String stormRoot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId);
String workerRoot = ConfigUtils.workerRoot(_conf, _workerId);
-
+
@SuppressWarnings("unchecked")
Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) _topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
List<String> blobFileNames = new ArrayList<>();
@@ -496,41 +460,44 @@ public abstract class Container implements Killable {
resourceFileNames.addAll(blobFileNames);
if (!_symlinksDisabled) {
- LOG.info("Creating symlinks for worker-id: {} storm-id: {} for files({}): {}", _workerId, _topologyId, resourceFileNames.size(), resourceFileNames);
+ LOG.info("Creating symlinks for worker-id: {} storm-id: {} for files({}): {}", _workerId, _topologyId, resourceFileNames.size(),
+ resourceFileNames);
if (targetResourcesDir.exists()) {
- _ops.createSymlink(new File(workerRoot, ServerConfigUtils.RESOURCES_SUBDIR), targetResourcesDir );
+ _ops.createSymlink(new File(workerRoot, ServerConfigUtils.RESOURCES_SUBDIR), targetResourcesDir);
} else {
- LOG.info("Topology jar for worker-id: {} storm-id: {} does not contain re sources directory {}." , _workerId, _topologyId, targetResourcesDir.toString() );
+ LOG.info("Topology jar for worker-id: {} storm-id: {} does not contain re sources directory {}.", _workerId, _topologyId,
+ targetResourcesDir.toString());
}
for (String fileName : blobFileNames) {
_ops.createSymlink(new File(workerRoot, fileName),
- new File(stormRoot, fileName));
+ new File(stormRoot, fileName));
}
} else if (blobFileNames.size() > 0) {
LOG.warn("Symlinks are disabled, no symlinks created for blobs {}", blobFileNames);
}
}
-
+
/**
* @return all of the pids that are a part of this container.
*/
protected Set<Long> getAllPids() throws IOException {
Set<Long> ret = new HashSet<>();
- for (String listing: ConfigUtils.readDirContents(ConfigUtils.workerPidsRoot(_conf, _workerId))) {
+ for (String listing : ConfigUtils.readDirContents(ConfigUtils.workerPidsRoot(_conf, _workerId))) {
ret.add(Long.valueOf(listing));
}
-
+
if (_resourceIsolationManager != null) {
Set<Long> morePids = _resourceIsolationManager.getRunningPids(_workerId);
- assert(morePids != null);
+ assert (morePids != null);
ret.addAll(morePids);
}
-
+
return ret;
}
-
- /**
+
+ /**
* @return the user that some operations should be done as.
+ *
* @throws IOException on any error
*/
protected String getWorkerUser() throws IOException {
@@ -552,40 +519,39 @@ public abstract class Container implements Killable {
throw new IllegalStateException("Could not recover the user for " + _workerId);
}
}
-
+
protected void saveWorkerUser(String user) throws IOException {
_type.assertFull();
LOG.info("SET worker-user {} {}", _workerId, user);
_ops.dump(new File(ConfigUtils.workerUserFile(_conf, _workerId)), user);
}
-
+
protected void deleteSavedWorkerUser() throws IOException {
LOG.info("REMOVE worker-user {}", _workerId);
_ops.deleteIfExists(new File(ConfigUtils.workerUserFile(_conf, _workerId)));
}
-
+
/**
- * Clean up the container partly preparing for restart.
- * By default delete all of the temp directories we are going
- * to get a new worker_id anyways.
- * POST CONDITION: the workerId will be set to null
+ * Clean up the container partly preparing for restart. By default delete all of the temp directories we are going to get a new
+ * worker_id anyways. POST CONDITION: the workerId will be set to null
+ *
* @throws IOException on any error
*/
public void cleanUpForRestart() throws IOException {
LOG.info("Cleaning up {}:{}", _supervisorId, _workerId);
Set<Long> pids = getAllPids();
String user = getWorkerUser();
-
+
for (Long pid : pids) {
File path = new File(ConfigUtils.workerPidPath(_conf, _workerId, pid));
_ops.deleteIfExists(path, user, _workerId);
}
-
+
//clean up for resource isolation if enabled
if (_resourceIsolationManager != null) {
_resourceIsolationManager.releaseResourcesForWorker(_workerId);
}
-
+
//Always make sure to clean up everything else before worker directory
//is removed since that is what is going to trigger the retry for cleanup
_ops.deleteIfExists(new File(ConfigUtils.workerHeartbeatsRoot(_conf, _workerId)), user, _workerId);
@@ -597,8 +563,9 @@ public abstract class Container implements Killable {
}
/**
- * Check if the container is over its memory limit AND needs to be killed. This does not necessarily mean
- * that it just went over the limit.
+ * Check if the container is over its memory limit AND needs to be killed. This does not necessarily mean that it just went over the
+ * limit.
+ *
* @throws IOException on any error
*/
public boolean isMemoryLimitViolated(LocalAssignment withUpdatedLimits) throws IOException {
@@ -674,16 +641,14 @@ public abstract class Container implements Killable {
}
/**
- * Launch the process for the first time.
- * PREREQUISITE: setup has run and passed
+ * Launch the process for the first time. PREREQUISITE: setup has run and passed
*
* @throws IOException on any error
*/
public abstract void launch() throws IOException;
/**
- * Restart the processes in this container.
- * PREREQUISITE: cleanUpForRestart has run and passed
+ * Restart the processes in this container. PREREQUISITE: cleanUpForRestart has run and passed
*
* @throws IOException on any error
*/
@@ -698,9 +663,10 @@ public abstract class Container implements Killable {
* Run a profiling request.
*
* @param request the request to run
- * @param stop is this a stop request?
+ * @param stop is this a stop request?
* @return true if it succeeded, else false
- * @throws IOException on any error
+ *
+ * @throws IOException on any error
* @throws InterruptedException if running the command is interrupted.
*/
public abstract boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException;
@@ -731,7 +697,7 @@ public abstract class Container implements Killable {
long timestamp = System.currentTimeMillis();
double value = _usedMemory.get(_port).memory;
WorkerMetricPoint workerMetric = new WorkerMetricPoint(MEMORY_USED_METRIC, timestamp, value, SYSTEM_COMPONENT_ID,
- INVALID_EXECUTOR_ID, INVALID_STREAM_ID);
+ INVALID_EXECUTOR_ID, INVALID_STREAM_ID);
WorkerMetricList metricList = new WorkerMetricList();
metricList.add_to_metrics(workerMetric);
@@ -751,4 +717,47 @@ public abstract class Container implements Killable {
this.lastMetricProcessTime = System.currentTimeMillis();
}
}
+
+ public static enum ContainerType {
+ LAUNCH(false, false),
+ RECOVER_FULL(true, false),
+ RECOVER_PARTIAL(true, true);
+
+ private final boolean _recovery;
+ private final boolean _onlyKillable;
+
+ ContainerType(boolean recovery, boolean onlyKillable) {
+ _recovery = recovery;
+ _onlyKillable = onlyKillable;
+ }
+
+ public boolean isRecovery() {
+ return _recovery;
+ }
+
+ public void assertFull() {
+ if (_onlyKillable) {
+ throw new IllegalStateException("Container is only Killable.");
+ }
+ }
+
+ public boolean isOnlyKillable() {
+ return _onlyKillable;
+ }
+ }
+
+ private static class TopoAndMemory {
+ public final String topoId;
+ public final long memory;
+
+ public TopoAndMemory(String id, long mem) {
+ topoId = id;
+ memory = mem;
+ }
+
+ @Override
+ public String toString() {
+ return "{TOPO: " + topoId + " at " + memory + " MB}";
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerLauncher.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerLauncher.java
index 527b321..2f32e38 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerLauncher.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerLauncher.java
@@ -1,33 +1,27 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.supervisor;
import java.io.IOException;
import java.util.Map;
-
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
import org.apache.storm.container.ResourceIsolationInterface;
import org.apache.storm.generated.LocalAssignment;
import org.apache.storm.messaging.IContext;
import org.apache.storm.utils.ConfigUtils;
-import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.ObjectReader;
import org.apache.storm.utils.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,9 +31,13 @@ import org.slf4j.LoggerFactory;
*/
public abstract class ContainerLauncher {
private static final Logger LOG = LoggerFactory.getLogger(ContainerLauncher.class);
-
+
+ protected ContainerLauncher() {
+ //Empty
+ }
+
/**
- * Factory to create the right container launcher
+ * Factory to create the right container launcher
* for the config and the environment.
* @param conf the config
* @param supervisorId the ID of the supervisor
@@ -49,16 +47,17 @@ public abstract class ContainerLauncher {
* @throws IOException on any error
*/
public static ContainerLauncher make(Map<String, Object> conf, String supervisorId, int supervisorPort,
- IContext sharedContext) throws IOException {
+ IContext sharedContext) throws IOException {
if (ConfigUtils.isLocalMode(conf)) {
return new LocalContainerLauncher(conf, supervisorId, supervisorPort, sharedContext);
}
-
+
ResourceIsolationInterface resourceIsolationManager = null;
if (ObjectReader.getBoolean(conf.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN_ENABLE), false)) {
resourceIsolationManager = ReflectionUtils.newInstance((String) conf.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN));
resourceIsolationManager.prepare(conf);
- LOG.info("Using resource isolation plugin {} {}", conf.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN), resourceIsolationManager);
+ LOG.info("Using resource isolation plugin {} {}", conf.get(DaemonConfig.STORM_RESOURCE_ISOLATION_PLUGIN),
+ resourceIsolationManager);
}
if (ObjectReader.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
@@ -66,10 +65,6 @@ public abstract class ContainerLauncher {
}
return new BasicContainerLauncher(conf, supervisorId, supervisorPort, resourceIsolationManager);
}
-
- protected ContainerLauncher() {
- //Empty
- }
/**
* Launch a container in a given slot
@@ -80,7 +75,7 @@ public abstract class ContainerLauncher {
* @throws IOException on any error
*/
public abstract Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException;
-
+
/**
* Recover a container for a running process
* @param port the port the assignment is running on
@@ -90,8 +85,9 @@ public abstract class ContainerLauncher {
* @throws IOException on any error
* @throws ContainerRecoveryException if the Container could not be recovered
*/
- public abstract Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException, ContainerRecoveryException;
-
+ public abstract Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException,
+ ContainerRecoveryException;
+
/**
* Try to recover a container using just the worker ID.
* The result is really only useful for killing the container
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java
index 7ab6e67..491ffdc 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.supervisor;
/**
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
index 8785f86..7b7ff1b 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.supervisor;
import org.apache.storm.utils.Utils;
@@ -23,6 +18,7 @@ import org.slf4j.LoggerFactory;
public class DefaultUncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
private static final Logger LOG = LoggerFactory.getLogger(DefaultUncaughtExceptionHandler.class);
+
@Override
public void uncaughtException(Thread t, Throwable e) {
LOG.error("Error when processing event", e);
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java
index 79df800..09a7bfb 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.supervisor;
import org.apache.storm.event.EventManager;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Killable.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Killable.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Killable.java
index 8d6d8e0..420f277 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Killable.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/Killable.java
@@ -1,46 +1,41 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.supervisor;
import java.io.IOException;
public interface Killable {
-
+
/**
* Kill the processes in this container nicely.
* kill -15 equivalent
* @throws IOException on any error
*/
public void kill() throws IOException;
-
+
/**
* Kill the processes in this container violently.
* kill -9 equivalent
* @throws IOException on any error
*/
public void forceKill() throws IOException;
-
+
/**
* @return true if all of the processes are dead, else false
* @throws IOException on any error
*/
public boolean areAllProcessesDead() throws IOException;
-
+
/**
* Clean up the container. It is not coming back.
* by default do the same thing as when restarting.
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainer.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainer.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainer.java
index 4afaffe..e972feb 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainer.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainer.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.supervisor;
import java.io.IOException;
import java.util.Map;
-
import org.apache.storm.ProcessSimulator;
import org.apache.storm.daemon.worker.Worker;
import org.apache.storm.generated.LocalAssignment;
@@ -31,11 +25,11 @@ import org.slf4j.LoggerFactory;
public class LocalContainer extends Container {
private static final Logger LOG = LoggerFactory.getLogger(LocalContainer.class);
- private volatile boolean _isAlive = false;
private final IContext _sharedContext;
-
+ private volatile boolean _isAlive = false;
+
public LocalContainer(Map<String, Object> conf, String supervisorId, int supervisorPort, int port,
- LocalAssignment assignment, IContext sharedContext) throws IOException {
+ LocalAssignment assignment, IContext sharedContext) throws IOException {
super(ContainerType.LAUNCH, conf, supervisorId, supervisorPort, port, assignment, null, null, null, null);
_sharedContext = sharedContext;
_workerId = Utils.uuid();
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java
index 1c43128..c2ff66f 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java
@@ -1,25 +1,19 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.supervisor;
import java.io.IOException;
import java.util.Map;
-
import org.apache.storm.generated.LocalAssignment;
import org.apache.storm.messaging.IContext;
import org.apache.storm.utils.LocalState;
@@ -34,7 +28,7 @@ public class LocalContainerLauncher extends ContainerLauncher {
private final IContext _sharedContext;
public LocalContainerLauncher(Map<String, Object> conf, String supervisorId, int supervisorPort,
- IContext sharedContext) {
+ IContext sharedContext) {
_conf = conf;
_supervisorId = supervisorId;
_supervisorPort = supervisorPort;
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/OnlyLatestExecutor.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/OnlyLatestExecutor.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/OnlyLatestExecutor.java
index 7dc9b0b..a0d35d9 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/OnlyLatestExecutor.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/OnlyLatestExecutor.java
@@ -25,8 +25,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * This allows you to submit a Runnable with a key. If the previous submission for that key has not yet run,
- * it will be replaced with the latest one.
+ * This allows you to submit a Runnable with a key. If the previous submission for that key has not yet run, it will be replaced with the
+ * latest one.
*/
public class OnlyLatestExecutor<K> {
private static final Logger LOG = LoggerFactory.getLogger(OnlyLatestExecutor.class);
@@ -40,8 +40,9 @@ public class OnlyLatestExecutor<K> {
/**
* Run something in the future, but replace it with the latest if it is taking too long
+ *
* @param key what to use to dedupe things.
- * @param r what you want to run.
+ * @param r what you want to run.
*/
public void execute(final K key, Runnable r) {
Runnable old = latest.put(key, r);
@@ -55,6 +56,6 @@ public class OnlyLatestExecutor<K> {
});
} else {
LOG.debug("Replacing runnable for {} - {}", key, r);
- }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ReadClusterState.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ReadClusterState.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ReadClusterState.java
index e065f38..b617345 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ReadClusterState.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/ReadClusterState.java
@@ -1,19 +1,13 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.daemon.supervisor;
@@ -28,7 +22,6 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-
import org.apache.storm.DaemonConfig;
import org.apache.storm.cluster.IStormClusterState;
import org.apache.storm.daemon.supervisor.Slot.MachineState;
@@ -51,7 +44,17 @@ import org.slf4j.LoggerFactory;
public class ReadClusterState implements Runnable, AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(ReadClusterState.class);
-
+ private static final long ERROR_MILLIS = 60_000; //1 min. This really means something is wrong. Even on a very slow node
+ public static final UniFunc<Slot> DEFAULT_ON_ERROR_TIMEOUT = (slot) -> {
+ throw new IllegalStateException("It took over " + ERROR_MILLIS + "ms to shut down slot " + slot);
+ };
+ public static final UniFunc<Slot> THREAD_DUMP_ON_ERROR = (slot) -> {
+ LOG.warn("Shutdown of slot {} appears to be stuck\n{}", slot, Utils.threadDump());
+ DEFAULT_ON_ERROR_TIMEOUT.call(slot);
+ };
+ private static final long WARN_MILLIS = 1_000; //Initial timeout 1 second. Workers commit suicide after this
+ public static final UniFunc<Slot> DEFAULT_ON_WARN_TIMEOUT =
+ (slot) -> LOG.warn("It has taken {}ms so far and {} is still not shut down.", WARN_MILLIS, slot);
private final Map<String, Object> superConf;
private final IStormClusterState stormClusterState;
private final Map<Integer, Slot> slots = new HashMap<>();
@@ -90,14 +93,14 @@ public class ReadClusterState implements Runnable, AutoCloseable {
}
@SuppressWarnings("unchecked")
- List<Number> ports = (List<Number>)superConf.get(DaemonConfig.SUPERVISOR_SLOTS_PORTS);
- for (Number port: ports) {
+ List<Number> ports = (List<Number>) superConf.get(DaemonConfig.SUPERVISOR_SLOTS_PORTS);
+ for (Number port : ports) {
slots.put(port.intValue(), mkSlot(port.intValue()));
}
-
+
try {
Collection<String> workers = SupervisorUtils.supervisorWorkerIds(superConf);
- for (Slot slot: slots.values()) {
+ for (Slot slot : slots.values()) {
String workerId = slot.getWorkerId();
if (workerId != null) {
workers.remove(workerId);
@@ -109,35 +112,35 @@ public class ReadClusterState implements Runnable, AutoCloseable {
} catch (Exception e) {
LOG.warn("Error trying to clean up old workers", e);
}
-
- for (Slot slot: slots.values()) {
+
+ for (Slot slot : slots.values()) {
slot.start();
}
}
private Slot mkSlot(int port) throws Exception {
return new Slot(localizer, superConf, launcher, host, port,
- localState, stormClusterState, iSuper, cachedAssignments, metricsExec, metricsProcessor);
+ localState, stormClusterState, iSuper, cachedAssignments, metricsExec, metricsProcessor);
}
-
+
@Override
public synchronized void run() {
try {
List<String> stormIds = stormClusterState.assignments(null);
Map<String, Assignment> assignmentsSnapshot = getAssignmentsSnapshot(stormClusterState);
-
+
Map<Integer, LocalAssignment> allAssignments = readAssignments(assignmentsSnapshot);
if (allAssignments == null) {
//Something odd happened try again later
return;
}
Map<String, List<ProfileRequest>> topoIdToProfilerActions = getProfileActions(stormClusterState, stormIds);
-
+
HashSet<Integer> assignedPorts = new HashSet<>();
LOG.debug("Synchronizing supervisor");
LOG.debug("All assignment: {}", allAssignments);
LOG.debug("Topology Ids -> Profiler Actions {}", topoIdToProfilerActions);
- for (Integer port: allAssignments.keySet()) {
+ for (Integer port : allAssignments.keySet()) {
if (iSuper.confirmAssigned(port)) {
assignedPorts.add(port);
}
@@ -145,12 +148,12 @@ public class ReadClusterState implements Runnable, AutoCloseable {
HashSet<Integer> allPorts = new HashSet<>(assignedPorts);
iSuper.assigned(allPorts);
allPorts.addAll(slots.keySet());
-
+
Map<Integer, Set<TopoProfileAction>> filtered = new HashMap<>();
- for (Entry<String, List<ProfileRequest>> entry: topoIdToProfilerActions.entrySet()) {
+ for (Entry<String, List<ProfileRequest>> entry : topoIdToProfilerActions.entrySet()) {
String topoId = entry.getKey();
if (entry.getValue() != null) {
- for (ProfileRequest req: entry.getValue()) {
+ for (ProfileRequest req : entry.getValue()) {
NodeInfo ni = req.get_nodeInfo();
if (host.equals(ni.get_node())) {
Long port = ni.get_port().iterator().next();
@@ -164,8 +167,8 @@ public class ReadClusterState implements Runnable, AutoCloseable {
}
}
}
-
- for (Integer port: allPorts) {
+
+ for (Integer port : allPorts) {
Slot slot = slots.get(port);
if (slot == null) {
slot = mkSlot(port);
@@ -175,18 +178,19 @@ public class ReadClusterState implements Runnable, AutoCloseable {
slot.setNewAssignment(allAssignments.get(port));
slot.addProfilerActions(filtered.get(port));
}
-
+
} catch (Exception e) {
LOG.error("Failed to Sync Supervisor", e);
throw new RuntimeException(e);
}
}
-
+
protected Map<String, Assignment> getAssignmentsSnapshot(IStormClusterState stormClusterState) throws Exception {
return stormClusterState.assignmentsInfo();
}
-
- protected Map<String, List<ProfileRequest>> getProfileActions(IStormClusterState stormClusterState, List<String> stormIds) throws Exception {
+
+ protected Map<String, List<ProfileRequest>> getProfileActions(IStormClusterState stormClusterState, List<String> stormIds) throws
+ Exception {
Map<String, List<ProfileRequest>> ret = new HashMap<String, List<ProfileRequest>>();
for (String stormId : stormIds) {
List<ProfileRequest> profileRequests = stormClusterState.getTopologyProfileRequests(stormId);
@@ -194,7 +198,7 @@ public class ReadClusterState implements Runnable, AutoCloseable {
}
return ret;
}
-
+
protected Map<Integer, LocalAssignment> readAssignments(Map<String, Assignment> assignmentsSnapshot) {
try {
Map<Integer, LocalAssignment> portLA = new HashMap<>();
@@ -214,7 +218,7 @@ public class ReadClusterState implements Runnable, AutoCloseable {
portLA.put(port, la);
} else {
throw new RuntimeException("Should not have multiple topologies assigned to one port "
- + port + " " + la + " " + portLA);
+ + port + " " + la + " " + portLA);
}
}
}
@@ -230,7 +234,7 @@ public class ReadClusterState implements Runnable, AutoCloseable {
return null;
}
}
-
+
protected Map<Integer, LocalAssignment> readMyExecutors(String topoId, String assignmentId, Assignment assignment) {
Map<Integer, LocalAssignment> portTasks = new HashMap<>();
Map<Long, WorkerResources> slotsResources = new HashMap<>();
@@ -276,7 +280,7 @@ public class ReadClusterState implements Runnable, AutoCloseable {
}
List<ExecutorInfo> executorInfoList = localAssignment.get_executors();
executorInfoList.add(new ExecutorInfo(entry.getKey().get(0).intValue(),
- entry.getKey().get(entry.getKey().size() - 1).intValue()));
+ entry.getKey().get(entry.getKey().size() - 1).intValue()));
}
}
}
@@ -284,34 +288,23 @@ public class ReadClusterState implements Runnable, AutoCloseable {
return portTasks;
}
- private static final long WARN_MILLIS = 1_000; //Initial timeout 1 second. Workers commit suicide after this
- private static final long ERROR_MILLIS = 60_000; //1 min. This really means something is wrong. Even on a very slow node
- public static final UniFunc<Slot> DEFAULT_ON_ERROR_TIMEOUT = (slot) -> {
- throw new IllegalStateException("It took over " + ERROR_MILLIS + "ms to shut down slot " + slot);
- };
- public static final UniFunc<Slot> DEFAULT_ON_WARN_TIMEOUT = (slot) -> LOG.warn("It has taken {}ms so far and {} is still not shut down.", WARN_MILLIS, slot);
- public static final UniFunc<Slot> THREAD_DUMP_ON_ERROR = (slot) -> {
- LOG.warn("Shutdown of slot {} appears to be stuck\n{}", slot, Utils.threadDump());
- DEFAULT_ON_ERROR_TIMEOUT.call(slot);
- };
-
public synchronized void shutdownAllWorkers(UniFunc<Slot> onWarnTimeout, UniFunc<Slot> onErrorTimeout) {
- for (Slot slot: slots.values()) {
+ for (Slot slot : slots.values()) {
LOG.info("Setting {} assignment to null", slot);
slot.setNewAssignment(null);
}
-
+
if (onWarnTimeout == null) {
onWarnTimeout = DEFAULT_ON_WARN_TIMEOUT;
}
-
+
if (onErrorTimeout == null) {
onErrorTimeout = DEFAULT_ON_ERROR_TIMEOUT;
}
-
+
long startTime = Time.currentTimeMillis();
Exception exp = null;
- for (Slot slot: slots.values()) {
+ for (Slot slot : slots.values()) {
LOG.info("Waiting for {} to be EMPTY, currently {}", slot, slot.getMachineState());
try {
while (slot.getMachineState() != MachineState.EMPTY) {
@@ -319,7 +312,7 @@ public class ReadClusterState implements Runnable, AutoCloseable {
if (timeSpentMillis > ERROR_MILLIS) {
onErrorTimeout.call(slot);
}
-
+
if (timeSpentMillis > WARN_MILLIS) {
onWarnTimeout.call(slot);
}
@@ -335,15 +328,15 @@ public class ReadClusterState implements Runnable, AutoCloseable {
}
if (exp != null) {
if (exp instanceof RuntimeException) {
- throw (RuntimeException)exp;
+ throw (RuntimeException) exp;
}
throw new RuntimeException(exp);
}
}
-
+
@Override
public void close() {
- for (Slot slot: slots.values()) {
+ for (Slot slot : slots.values()) {
try {
slot.close();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/RunAsUserContainer.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/RunAsUserContainer.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/RunAsUserContainer.java
index 2559d7a..eadd635 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/RunAsUserContainer.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/RunAsUserContainer.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.daemon.supervisor;
import java.io.File;
@@ -22,12 +23,11 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-
import org.apache.storm.container.ResourceIsolationInterface;
import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.utils.LocalState;
import org.apache.storm.utils.ServerUtils;
import org.apache.storm.utils.Utils;
-import org.apache.storm.utils.LocalState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,15 +39,15 @@ public class RunAsUserContainer extends BasicContainer {
ResourceIsolationInterface resourceIsolationManager, LocalState localState,
String workerId) throws IOException {
this(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, localState, workerId,
- null, null, null);
+ null, null, null);
}
-
+
RunAsUserContainer(Container.ContainerType type, Map<String, Object> conf, String supervisorId, int supervisorPort,
int port, LocalAssignment assignment, ResourceIsolationInterface resourceIsolationManager,
LocalState localState, String workerId, Map<String, Object> topoConf, AdvancedFSOps ops,
String profileCmd) throws IOException {
super(type, conf, supervisorId, supervisorPort, port, assignment, resourceIsolationManager, localState,
- workerId, topoConf, ops, profileCmd);
+ workerId, topoConf, ops, profileCmd);
if (Utils.isOnWindows()) {
throw new UnsupportedOperationException("ERROR: Windows doesn't support running workers as different users yet");
}
@@ -56,22 +56,23 @@ public class RunAsUserContainer extends BasicContainer {
private void signal(long pid, int signal) throws IOException {
List<String> commands = Arrays.asList("signal", String.valueOf(pid), String.valueOf(signal));
String user = getWorkerUser();
- String logPrefix = "kill -"+signal+" " + pid;
+ String logPrefix = "kill -" + signal + " " + pid;
ClientSupervisorUtils.processLauncherAndWait(_conf, user, commands, null, logPrefix);
}
-
+
@Override
protected void kill(long pid) throws IOException {
signal(pid, 15);
}
-
+
@Override
protected void forceKill(long pid) throws IOException {
signal(pid, 9);
}
-
+
@Override
- protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix, File targetDir) throws IOException, InterruptedException {
+ protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix, File targetDir) throws
+ IOException, InterruptedException {
String user = this.getWorkerUser();
String td = targetDir.getAbsolutePath();
LOG.info("Running as user: {} command: {}", user, command);
@@ -90,8 +91,8 @@ public class RunAsUserContainer extends BasicContainer {
}
@Override
- protected void launchWorkerProcess(List<String> command, Map<String, String> env,
- String logPrefix, ExitCodeCallback processExitCallback, File targetDir) throws IOException {
+ protected void launchWorkerProcess(List<String> command, Map<String, String> env,
+ String logPrefix, ExitCodeCallback processExitCallback, File targetDir) throws IOException {
String workerDir = targetDir.getAbsolutePath();
String user = this.getWorkerUser();
List<String> args = Arrays.asList("worker", workerDir, ServerUtils.writeScript(workerDir, command, env));
http://git-wip-us.apache.org/repos/asf/storm/blob/f1c0bcbe/storm-server/src/main/java/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java
index e6439db..c0bb47f 100644
--- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java
+++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java
@@ -1,38 +1,32 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.daemon.supervisor;
import java.io.IOException;
import java.util.Map;
-
import org.apache.storm.container.ResourceIsolationInterface;
import org.apache.storm.daemon.supervisor.Container.ContainerType;
import org.apache.storm.generated.LocalAssignment;
import org.apache.storm.utils.LocalState;
public class RunAsUserContainerLauncher extends ContainerLauncher {
+ protected final ResourceIsolationInterface _resourceIsolationManager;
private final Map<String, Object> _conf;
private final String _supervisorId;
private final int _supervisorPort;
- protected final ResourceIsolationInterface _resourceIsolationManager;
-
+
public RunAsUserContainerLauncher(Map<String, Object> conf, String supervisorId, int supervisorPort,
- ResourceIsolationInterface resourceIsolationManager) throws IOException {
+ ResourceIsolationInterface resourceIsolationManager) throws IOException {
_conf = conf;
_supervisorId = supervisorId;
_supervisorPort = supervisorPort;
@@ -42,7 +36,7 @@ public class RunAsUserContainerLauncher extends ContainerLauncher {
@Override
public Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
Container container = new RunAsUserContainer(ContainerType.LAUNCH, _conf, _supervisorId, _supervisorPort, port,
- assignment, _resourceIsolationManager, state, null, null, null, null);
+ assignment, _resourceIsolationManager, state, null, null, null, null);
container.setup();
container.launch();
return container;
@@ -51,13 +45,13 @@ public class RunAsUserContainerLauncher extends ContainerLauncher {
@Override
public Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
return new RunAsUserContainer(ContainerType.RECOVER_FULL, _conf, _supervisorId, _supervisorPort, port,
- assignment, _resourceIsolationManager, state, null, null, null, null);
+ assignment, _resourceIsolationManager, state, null, null, null, null);
}
-
+
@Override
public Killable recoverContainer(String workerId, LocalState localState) throws IOException {
return new RunAsUserContainer(ContainerType.RECOVER_PARTIAL, _conf, _supervisorId, _supervisorPort, -1, null,
- _resourceIsolationManager, localState, workerId, null, null, null);
+ _resourceIsolationManager, localState, workerId, null, null, null);
}
}
[44/63] [abbrv] storm git commit: Fixing stylecheck problems with
maven-shade-clojure-transformer
Posted by bo...@apache.org.
Fixing stylecheck problems with maven-shade-clojure-transformer
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6ccf6a0a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6ccf6a0a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6ccf6a0a
Branch: refs/heads/master
Commit: 6ccf6a0a0954590e3db4c95a3f22b504a5a72757
Parents: 84084ab
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 23:05:45 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 00:22:37 2018 -0400
----------------------------------------------------------------------
.../maven-shade-clojure-transformer/pom.xml | 2 +-
.../storm/maven/shade/clojure/ClojureTransformer.java | 14 +++++++-------
2 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6ccf6a0a/storm-buildtools/maven-shade-clojure-transformer/pom.xml
----------------------------------------------------------------------
diff --git a/storm-buildtools/maven-shade-clojure-transformer/pom.xml b/storm-buildtools/maven-shade-clojure-transformer/pom.xml
index cdb6fa8..b1e5616 100644
--- a/storm-buildtools/maven-shade-clojure-transformer/pom.xml
+++ b/storm-buildtools/maven-shade-clojure-transformer/pom.xml
@@ -42,7 +42,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>16</maxAllowedViolations>
+ <maxAllowedViolations>0</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/6ccf6a0a/storm-buildtools/maven-shade-clojure-transformer/src/main/java/org/apache/storm/maven/shade/clojure/ClojureTransformer.java
----------------------------------------------------------------------
diff --git a/storm-buildtools/maven-shade-clojure-transformer/src/main/java/org/apache/storm/maven/shade/clojure/ClojureTransformer.java b/storm-buildtools/maven-shade-clojure-transformer/src/main/java/org/apache/storm/maven/shade/clojure/ClojureTransformer.java
index a24c67d..c2e6590 100644
--- a/storm-buildtools/maven-shade-clojure-transformer/src/main/java/org/apache/storm/maven/shade/clojure/ClojureTransformer.java
+++ b/storm-buildtools/maven-shade-clojure-transformer/src/main/java/org/apache/storm/maven/shade/clojure/ClojureTransformer.java
@@ -15,10 +15,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.storm.maven.shade.clojure;
-import org.apache.maven.plugins.shade.relocation.Relocator;
-import org.apache.maven.plugins.shade.resource.ResourceTransformer;
+package org.apache.storm.maven.shade.clojure;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -27,6 +25,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
+import org.apache.maven.plugins.shade.relocation.Relocator;
+import org.apache.maven.plugins.shade.resource.ResourceTransformer;
public class ClojureTransformer implements ResourceTransformer {
@@ -35,7 +35,7 @@ public class ClojureTransformer implements ResourceTransformer {
@Override
public boolean canTransformResource(String s) {
- if(s.endsWith(".clj")){
+ if (s.endsWith(".clj")) {
return true;
}
return false;
@@ -46,12 +46,12 @@ public class ClojureTransformer implements ResourceTransformer {
ByteArrayOutputStream out = new ByteArrayOutputStream();
int b;
- while((b = inputStream.read()) != -1){
+ while ((b = inputStream.read()) != -1) {
out.write(b);
}
String data = out.toString();
- for(Relocator rel : relocators){
+ for (Relocator rel : relocators) {
data = rel.applyToSourceContent(data);
}
this.entries.put(s, data);
@@ -64,7 +64,7 @@ public class ClojureTransformer implements ResourceTransformer {
@Override
public void modifyOutputStream(JarOutputStream jarOut) throws IOException {
- for(String key : this.entries.keySet()){
+ for (String key : this.entries.keySet()) {
jarOut.putNextEntry(new JarEntry(key));
jarOut.write(this.entries.get(key).getBytes());
}
[43/63] [abbrv] storm git commit: Fixing stylecheck problems with
storm-maven-plugins
Posted by bo...@apache.org.
Fixing stylecheck problems with storm-maven-plugins
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/84084ab0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/84084ab0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/84084ab0
Branch: refs/heads/master
Commit: 84084ab0ac16390f24e7f24a1d9d1693062cb023
Parents: 4fe4f04
Author: Kishor Patil <kp...@yahoo-inc.com>
Authored: Sun Apr 22 23:03:52 2018 -0400
Committer: Kishor Patil <kp...@yahoo-inc.com>
Committed: Mon Apr 23 00:22:37 2018 -0400
----------------------------------------------------------------------
storm-buildtools/storm-maven-plugins/pom.xml | 2 +-
.../storm/maven/plugin/util/CommandExec.java | 118 +++--
.../plugin/versioninfo/VersionInfoMojo.java | 457 +++++++++----------
3 files changed, 282 insertions(+), 295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/84084ab0/storm-buildtools/storm-maven-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/storm-buildtools/storm-maven-plugins/pom.xml b/storm-buildtools/storm-maven-plugins/pom.xml
index 3f0a27d..b850f21 100644
--- a/storm-buildtools/storm-maven-plugins/pom.xml
+++ b/storm-buildtools/storm-maven-plugins/pom.xml
@@ -81,7 +81,7 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<!--Note - the version would be inherited-->
<configuration>
- <maxAllowedViolations>269</maxAllowedViolations>
+ <maxAllowedViolations>11</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/storm/blob/84084ab0/storm-buildtools/storm-maven-plugins/src/main/java/org/apache/storm/maven/plugin/util/CommandExec.java
----------------------------------------------------------------------
diff --git a/storm-buildtools/storm-maven-plugins/src/main/java/org/apache/storm/maven/plugin/util/CommandExec.java b/storm-buildtools/storm-maven-plugins/src/main/java/org/apache/storm/maven/plugin/util/CommandExec.java
index c289f51..8bad082 100644
--- a/storm-buildtools/storm-maven-plugins/src/main/java/org/apache/storm/maven/plugin/util/CommandExec.java
+++ b/storm-buildtools/storm-maven-plugins/src/main/java/org/apache/storm/maven/plugin/util/CommandExec.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.maven.plugin.util;
import java.io.BufferedReader;
@@ -23,67 +18,66 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
-
import org.apache.maven.plugin.Mojo;
public class CommandExec {
- private Mojo mojo;
+ private Mojo mojo;
- public CommandExec(Mojo mojo) {
- this.mojo = mojo;
- }
+ public CommandExec(Mojo mojo) {
+ this.mojo = mojo;
+ }
- public int run(List<String> command, List<String> output) {
- int retCode = 1;
- ProcessBuilder pb = new ProcessBuilder(command);
- try {
- Process p = pb.start();
- OutputBufferThread stdOut = new OutputBufferThread(p.getInputStream());
- OutputBufferThread stdErr = new OutputBufferThread(p.getErrorStream());
- stdOut.start();
- stdErr.start();
- retCode = p.waitFor();
- if (retCode != 0) {
- mojo.getLog().warn(command + " failed with error code " + retCode);
- for (String s : stdErr.getOutput()) {
- mojo.getLog().debug(s);
+ public int run(List<String> command, List<String> output) {
+ int retCode = 1;
+ ProcessBuilder pb = new ProcessBuilder(command);
+ try {
+ Process p = pb.start();
+ OutputBufferThread stdOut = new OutputBufferThread(p.getInputStream());
+ OutputBufferThread stdErr = new OutputBufferThread(p.getErrorStream());
+ stdOut.start();
+ stdErr.start();
+ retCode = p.waitFor();
+ if (retCode != 0) {
+ mojo.getLog().warn(command + " failed with error code " + retCode);
+ for (String s : stdErr.getOutput()) {
+ mojo.getLog().debug(s);
+ }
+ }
+ stdOut.join();
+ stdErr.join();
+ output.addAll(stdOut.getOutput());
+ } catch (Exception ex) {
+ mojo.getLog().warn(command + " failed: " + ex.toString());
}
- }
- stdOut.join();
- stdErr.join();
- output.addAll(stdOut.getOutput());
- } catch (Exception ex) {
- mojo.getLog().warn(command + " failed: " + ex.toString());
+ return retCode;
}
- return retCode;
- }
- private static class OutputBufferThread extends Thread {
- private List<String> output;
- private BufferedReader reader;
+ private static class OutputBufferThread extends Thread {
+ private List<String> output;
+ private BufferedReader reader;
- public OutputBufferThread(InputStream is) {
- this.setDaemon(true);
- output = new ArrayList<String>();
- reader = new BufferedReader(new InputStreamReader(is));
- }
+ public OutputBufferThread(InputStream is) {
+ this.setDaemon(true);
+ output = new ArrayList<String>();
+ reader = new BufferedReader(new InputStreamReader(is));
+ }
- @Override
- public void run() {
- try {
- String line = reader.readLine();
- while (line != null) {
- output.add(line);
- line = reader.readLine();
+ @Override
+ public void run() {
+ try {
+ String line = reader.readLine();
+ while (line != null) {
+ output.add(line);
+ line = reader.readLine();
+ }
+ } catch (IOException ex) {
+ throw new RuntimeException("make failed with error code "
+ + ex.toString());
+ }
}
- } catch (IOException ex) {
- throw new RuntimeException("make failed with error code "
- + ex.toString());
- }
- }
- public List<String> getOutput() {
- return output;
+ public List<String> getOutput() {
+ return output;
+ }
}
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/84084ab0/storm-buildtools/storm-maven-plugins/src/main/java/org/apache/storm/maven/plugin/versioninfo/VersionInfoMojo.java
----------------------------------------------------------------------
diff --git a/storm-buildtools/storm-maven-plugins/src/main/java/org/apache/storm/maven/plugin/versioninfo/VersionInfoMojo.java b/storm-buildtools/storm-maven-plugins/src/main/java/org/apache/storm/maven/plugin/versioninfo/VersionInfoMojo.java
index 31354e3..9eb0298 100644
--- a/storm-buildtools/storm-maven-plugins/src/main/java/org/apache/storm/maven/plugin/versioninfo/VersionInfoMojo.java
+++ b/storm-buildtools/storm-maven-plugins/src/main/java/org/apache/storm/maven/plugin/versioninfo/VersionInfoMojo.java
@@ -1,20 +1,15 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
+
package org.apache.storm.maven.plugin.versioninfo;
import java.io.File;
@@ -31,7 +26,6 @@ import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
-
import org.apache.maven.model.FileSet;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
@@ -44,261 +38,260 @@ import org.codehaus.plexus.util.FileUtils;
@Mojo(name = "version-info")
public class VersionInfoMojo extends AbstractMojo {
- @Parameter(defaultValue = "${project}")
- private MavenProject project;
+ @Parameter(defaultValue = "${project}")
+ private MavenProject project;
- @Parameter(required = true)
- private FileSet source;
+ @Parameter(required = true)
+ private FileSet source;
- @Parameter(defaultValue = "version-info.build.time")
- private String buildTimeProperty;
+ @Parameter(defaultValue = "version-info.build.time")
+ private String buildTimeProperty;
- @Parameter(defaultValue = "version-info.source.md5")
- private String md5Property;
+ @Parameter(defaultValue = "version-info.source.md5")
+ private String md5Property;
- @Parameter(defaultValue = "version-info.scm.uri")
- private String scmUriProperty;
+ @Parameter(defaultValue = "version-info.scm.uri")
+ private String scmUriProperty;
- @Parameter(defaultValue = "version-info.scm.branch")
- private String scmBranchProperty;
+ @Parameter(defaultValue = "version-info.scm.branch")
+ private String scmBranchProperty;
- @Parameter(defaultValue = "version-info.scm.commit")
- private String scmCommitProperty;
+ @Parameter(defaultValue = "version-info.scm.commit")
+ private String scmCommitProperty;
- @Parameter(defaultValue = "git")
- private String gitCommand;
+ @Parameter(defaultValue = "git")
+ private String gitCommand;
- @Parameter(defaultValue = "svn")
- private String svnCommand;
+ @Parameter(defaultValue = "svn")
+ private String svnCommand;
+ private List<String> scmOut;
- private enum SCM {
- NONE, SVN, GIT
- }
+ @SuppressWarnings("rawtypes")
+ private static String getCommaSeparatedList(List list) {
+ StringBuilder buffer = new StringBuilder();
+ String separator = "";
+ for (Object e : list) {
+ buffer.append(separator).append(e);
+ separator = ",";
+ }
+ return buffer.toString();
+ }
- @Override
- public void execute() throws MojoExecutionException {
- try {
- SCM scm = determineSCM();
- project.getProperties().setProperty(buildTimeProperty, getBuildTime());
- project.getProperties().setProperty(scmUriProperty, getSCMUri(scm));
- project.getProperties().setProperty(scmBranchProperty, getSCMBranch(scm));
- project.getProperties().setProperty(scmCommitProperty, getSCMCommit(scm));
- project.getProperties().setProperty(md5Property, computeMD5());
- } catch (Throwable ex) {
- throw new MojoExecutionException(ex.toString(), ex);
+ @SuppressWarnings("unchecked")
+ public static List<File> convertFileSetToFiles(FileSet source)
+ throws IOException {
+ String includes = getCommaSeparatedList(source.getIncludes());
+ String excludes = getCommaSeparatedList(source.getExcludes());
+ return FileUtils.getFiles(new File(source.getDirectory()), includes,
+ excludes);
}
- }
- private String getBuildTime() {
- DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
- dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- return dateFormat.format(new Date());
- }
+ @Override
+ public void execute() throws MojoExecutionException {
+ try {
+ SCM scm = determineSCM();
+ project.getProperties().setProperty(buildTimeProperty, getBuildTime());
+ project.getProperties().setProperty(scmUriProperty, getSCMUri(scm));
+ project.getProperties().setProperty(scmBranchProperty, getSCMBranch(scm));
+ project.getProperties().setProperty(scmCommitProperty, getSCMCommit(scm));
+ project.getProperties().setProperty(md5Property, computeMD5());
+ } catch (Throwable ex) {
+ throw new MojoExecutionException(ex.toString(), ex);
+ }
+ }
- private List<String> scmOut;
+ private String getBuildTime() {
+ DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'");
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ return dateFormat.format(new Date());
+ }
- /**
- * Determines which SCM is in use (Subversion, git, or none) and captures
- * output of the SCM command for later parsing.
- *
- * @return SCM in use for this build
- * @throws Exception if any error occurs attempting to determine SCM
- */
- private SCM determineSCM() throws Exception {
- CommandExec exec = new CommandExec(this);
- SCM scm = SCM.NONE;
- scmOut = new ArrayList<String>();
- int ret = exec.run(Arrays.asList(svnCommand, "info"), scmOut);
- if (ret == 0) {
- scm = SCM.SVN;
- } else {
- ret = exec.run(Arrays.asList(gitCommand, "branch"), scmOut);
- if (ret == 0) {
- ret = exec.run(Arrays.asList(gitCommand, "remote", "-v"), scmOut);
- if (ret != 0) {
- scm = SCM.NONE;
- scmOut = null;
+ /**
+ * Determines which SCM is in use (Subversion, git, or none) and captures
+ * output of the SCM command for later parsing.
+ *
+ * @return SCM in use for this build
+ * @throws Exception if any error occurs attempting to determine SCM
+ */
+ private SCM determineSCM() throws Exception {
+ CommandExec exec = new CommandExec(this);
+ SCM scm = SCM.NONE;
+ scmOut = new ArrayList<String>();
+ int ret = exec.run(Arrays.asList(svnCommand, "info"), scmOut);
+ if (ret == 0) {
+ scm = SCM.SVN;
} else {
- ret = exec.run(Arrays.asList(gitCommand, "log", "-n", "1"), scmOut);
- if (ret != 0) {
- scm = SCM.NONE;
- scmOut = null;
- } else {
- scm = SCM.GIT;
- }
+ ret = exec.run(Arrays.asList(gitCommand, "branch"), scmOut);
+ if (ret == 0) {
+ ret = exec.run(Arrays.asList(gitCommand, "remote", "-v"), scmOut);
+ if (ret != 0) {
+ scm = SCM.NONE;
+ scmOut = null;
+ } else {
+ ret = exec.run(Arrays.asList(gitCommand, "log", "-n", "1"), scmOut);
+ if (ret != 0) {
+ scm = SCM.NONE;
+ scmOut = null;
+ } else {
+ scm = SCM.GIT;
+ }
+ }
+ }
}
- }
- }
- if (scmOut != null) {
- getLog().debug(scmOut.toString());
+ if (scmOut != null) {
+ getLog().debug(scmOut.toString());
+ }
+ getLog().info("SCM: " + scm);
+ return scm;
}
- getLog().info("SCM: " + scm);
- return scm;
- }
- private String[] getSvnUriInfo(String str) {
- String[] res = new String[] { "Unknown", "Unknown" };
- try {
- String path = str;
- int index = path.indexOf("trunk");
- if (index > -1) {
- res[0] = path.substring(0, index - 1);
- res[1] = "trunk";
- } else {
- index = path.indexOf("branches");
- if (index > -1) {
- res[0] = path.substring(0, index - 1);
- int branchIndex = index + "branches".length() + 1;
- index = path.indexOf("/", branchIndex);
- if (index > -1) {
- res[1] = path.substring(branchIndex, index);
- } else {
- res[1] = path.substring(branchIndex);
- }
+ private String[] getSvnUriInfo(String str) {
+ String[] res = new String[]{ "Unknown", "Unknown" };
+ try {
+ String path = str;
+ int index = path.indexOf("trunk");
+ if (index > -1) {
+ res[0] = path.substring(0, index - 1);
+ res[1] = "trunk";
+ } else {
+ index = path.indexOf("branches");
+ if (index > -1) {
+ res[0] = path.substring(0, index - 1);
+ int branchIndex = index + "branches".length() + 1;
+ index = path.indexOf("/", branchIndex);
+ if (index > -1) {
+ res[1] = path.substring(branchIndex, index);
+ } else {
+ res[1] = path.substring(branchIndex);
+ }
+ }
+ }
+ } catch (Exception ex) {
+ getLog().warn("Could not determine URI & branch from SVN URI: " + str);
}
- }
- } catch (Exception ex) {
- getLog().warn("Could not determine URI & branch from SVN URI: " + str);
+ return res;
}
- return res;
- }
- private String getSCMUri(SCM scm) {
- String uri = "Unknown";
- switch (scm) {
- case SVN:
- for (String s : scmOut) {
- if (s.startsWith("URL:")) {
- uri = s.substring(4).trim();
- uri = getSvnUriInfo(uri)[0];
- break;
+ private String getSCMUri(SCM scm) {
+ String uri = "Unknown";
+ switch (scm) {
+ case SVN:
+ for (String s : scmOut) {
+ if (s.startsWith("URL:")) {
+ uri = s.substring(4).trim();
+ uri = getSvnUriInfo(uri)[0];
+ break;
+ }
+ }
+ break;
+ case GIT:
+ for (String s : scmOut) {
+ if (s.startsWith("origin") && s.endsWith("(fetch)")) {
+ uri = s.substring("origin".length());
+ uri = uri.substring(0, uri.length() - "(fetch)".length());
+ break;
+ }
+ }
+ break;
}
- }
- break;
- case GIT:
- for (String s : scmOut) {
- if (s.startsWith("origin") && s.endsWith("(fetch)")) {
- uri = s.substring("origin".length());
- uri = uri.substring(0, uri.length() - "(fetch)".length());
- break;
- }
- }
- break;
+ return uri.trim();
}
- return uri.trim();
- }
- private String getSCMCommit(SCM scm) {
- String commit = "Unknown";
- switch (scm) {
- case SVN:
- for (String s : scmOut) {
- if (s.startsWith("Revision:")) {
- commit = s.substring("Revision:".length());
- break;
- }
- }
- break;
- case GIT:
- for (String s : scmOut) {
- if (s.startsWith("commit")) {
- commit = s.substring("commit".length());
- break;
+ private String getSCMCommit(SCM scm) {
+ String commit = "Unknown";
+ switch (scm) {
+ case SVN:
+ for (String s : scmOut) {
+ if (s.startsWith("Revision:")) {
+ commit = s.substring("Revision:".length());
+ break;
+ }
+ }
+ break;
+ case GIT:
+ for (String s : scmOut) {
+ if (s.startsWith("commit")) {
+ commit = s.substring("commit".length());
+ break;
+ }
+ }
+ break;
}
- }
- break;
+ return commit.trim();
}
- return commit.trim();
- }
- private String getSCMBranch(SCM scm) {
- String branch = "Unknown";
- switch (scm) {
- case SVN:
- for (String s : scmOut) {
- if (s.startsWith("URL:")) {
- branch = s.substring(4).trim();
- branch = getSvnUriInfo(branch)[1];
- break;
+ private String getSCMBranch(SCM scm) {
+ String branch = "Unknown";
+ switch (scm) {
+ case SVN:
+ for (String s : scmOut) {
+ if (s.startsWith("URL:")) {
+ branch = s.substring(4).trim();
+ branch = getSvnUriInfo(branch)[1];
+ break;
+ }
+ }
+ break;
+ case GIT:
+ for (String s : scmOut) {
+ if (s.startsWith("*")) {
+ branch = s.substring("*".length());
+ break;
+ }
+ }
+ break;
}
- }
- break;
- case GIT:
- for (String s : scmOut) {
- if (s.startsWith("*")) {
- branch = s.substring("*".length());
- break;
- }
- }
- break;
+ return branch.trim();
}
- return branch.trim();
- }
-
- private byte[] readFile(File file) throws IOException {
- RandomAccessFile raf = new RandomAccessFile(file, "r");
- byte[] buffer = new byte[(int) raf.length()];
- raf.readFully(buffer);
- raf.close();
- return buffer;
- }
- private byte[] computeMD5(List<File> files) throws IOException,
- NoSuchAlgorithmException {
- MessageDigest md5 = MessageDigest.getInstance("MD5");
- for (File file : files) {
- getLog().debug("Computing MD5 for: " + file);
- md5.update(readFile(file));
+ private byte[] readFile(File file) throws IOException {
+ RandomAccessFile raf = new RandomAccessFile(file, "r");
+ byte[] buffer = new byte[(int) raf.length()];
+ raf.readFully(buffer);
+ raf.close();
+ return buffer;
}
- return md5.digest();
- }
- private String byteArrayToString(byte[] array) {
- StringBuilder sb = new StringBuilder();
- for (byte b : array) {
- sb.append(Integer.toHexString(0xff & b));
+ private byte[] computeMD5(List<File> files) throws IOException,
+ NoSuchAlgorithmException {
+ MessageDigest md5 = MessageDigest.getInstance("MD5");
+ for (File file : files) {
+ getLog().debug("Computing MD5 for: " + file);
+ md5.update(readFile(file));
+ }
+ return md5.digest();
}
- return sb.toString();
- }
- private String computeMD5() throws Exception {
- List<File> files = convertFileSetToFiles(source);
- // File order of MD5 calculation is significant. Sorting is done on
- // unix-format names, case-folded, in order to get a platform-independent
- // sort and calculate the same MD5 on all platforms.
- Collections.sort(files, new Comparator<File>() {
- @Override
- public int compare(File lhs, File rhs) {
- return normalizePath(lhs).compareTo(normalizePath(rhs));
- }
+ private String byteArrayToString(byte[] array) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : array) {
+ sb.append(Integer.toHexString(0xff & b));
+ }
+ return sb.toString();
+ }
- private String normalizePath(File file) {
- return file.getPath().toUpperCase().replaceAll("\\\\", "/");
- }
- });
- byte[] md5 = computeMD5(files);
- String md5str = byteArrayToString(md5);
- getLog().info("Computed MD5: " + md5str);
- return md5str;
- }
+ private String computeMD5() throws Exception {
+ List<File> files = convertFileSetToFiles(source);
+ // File order of MD5 calculation is significant. Sorting is done on
+ // unix-format names, case-folded, in order to get a platform-independent
+ // sort and calculate the same MD5 on all platforms.
+ Collections.sort(files, new Comparator<File>() {
+ @Override
+ public int compare(File lhs, File rhs) {
+ return normalizePath(lhs).compareTo(normalizePath(rhs));
+ }
- @SuppressWarnings("rawtypes")
- private static String getCommaSeparatedList(List list) {
- StringBuilder buffer = new StringBuilder();
- String separator = "";
- for (Object e : list) {
- buffer.append(separator).append(e);
- separator = ",";
+ private String normalizePath(File file) {
+ return file.getPath().toUpperCase().replaceAll("\\\\", "/");
+ }
+ });
+ byte[] md5 = computeMD5(files);
+ String md5str = byteArrayToString(md5);
+ getLog().info("Computed MD5: " + md5str);
+ return md5str;
}
- return buffer.toString();
- }
- @SuppressWarnings("unchecked")
- public static List<File> convertFileSetToFiles(FileSet source)
- throws IOException {
- String includes = getCommaSeparatedList(source.getIncludes());
- String excludes = getCommaSeparatedList(source.getExcludes());
- return FileUtils.getFiles(new File(source.getDirectory()), includes,
- excludes);
- }
+ private enum SCM {
+ NONE, SVN, GIT
+ }
}