You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2016/10/27 16:05:22 UTC
[01/21] storm git commit: STORM-2018: Supervisor V2
Repository: storm
Updated Branches:
refs/heads/1.x-branch e59447bd7 -> 17218ea87
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
new file mode 100644
index 0000000..9b20ef8
--- /dev/null
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
@@ -0,0 +1,265 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.storm.Config;
+import org.apache.storm.daemon.supervisor.Container.ContainerType;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileRequest;
+import org.junit.Test;
+import org.yaml.snakeyaml.Yaml;
+
+import com.google.common.base.Joiner;
+
+public class ContainerTest {
+ public static class MockContainer extends Container {
+
+ protected MockContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+ LocalAssignment assignment, String workerId,
+ Map<String, Object> topoConf, AdvancedFSOps ops) throws IOException {
+ super(type, conf, supervisorId, port, assignment, workerId, topoConf, ops);
+ }
+
+ public final List<Long> killedPids = new ArrayList<>();
+ public final List<Long> forceKilledPids = new ArrayList<>();
+ public final Set<Long> allPids = new HashSet<>();
+
+ @Override
+ protected void kill(long pid) {
+ killedPids.add(pid);
+ }
+
+ @Override
+ protected void forceKill(long pid) {
+ forceKilledPids.add(pid);
+ }
+
+ @Override
+ protected Set<Long> getAllPids() throws IOException {
+ return allPids;
+ }
+
+ @Override
+ public void launch() throws IOException {
+ fail("THIS IS NOT UNDER TEST");
+ }
+
+ @Override
+ public void relaunch() throws IOException {
+ fail("THIS IS NOT UNDER TEST");
+ }
+
+ @Override
+ public boolean didMainProcessExit() {
+ fail("THIS IS NOT UNDER TEST");
+ return false;
+ }
+
+ @Override
+ public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException {
+ fail("THIS IS NOT UNDER TEST");
+ return false;
+ }
+ }
+
+ @Test
+ public void testKill() throws Exception {
+ final String topoId = "test_topology";
+ final Map<String, Object> superConf = new HashMap<>();
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+ MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", 8080, la, "worker", new HashMap<>(), ops);
+ mc.kill();
+ assertEquals(Collections.EMPTY_LIST, mc.killedPids);
+ assertEquals(Collections.EMPTY_LIST, mc.forceKilledPids);
+ mc.forceKill();
+ assertEquals(Collections.EMPTY_LIST, mc.killedPids);
+ assertEquals(Collections.EMPTY_LIST, mc.forceKilledPids);
+
+ long pid = 987654321;
+ mc.allPids.add(pid);
+
+ mc.kill();
+ assertEquals(mc.allPids, new HashSet<>(mc.killedPids));
+ assertEquals(Collections.EMPTY_LIST, mc.forceKilledPids);
+ mc.killedPids.clear();
+
+ mc.forceKill();
+ assertEquals(Collections.EMPTY_LIST, mc.killedPids);
+ assertEquals(mc.allPids, new HashSet<>(mc.forceKilledPids));
+ }
+
+ private static final Joiner PATH_JOIN = Joiner.on(File.separator).skipNulls();
+ private static final String DOUBLE_SEP = File.separator + File.separator;
+ static String asAbsPath(String ... parts) {
+ return (File.separator + PATH_JOIN.join(parts)).replace(DOUBLE_SEP, File.separator);
+ }
+
+ static File asAbsFile(String ... parts) {
+ return new File(asAbsPath(parts));
+ }
+
+ static String asPath(String ... parts) {
+ return PATH_JOIN.join(parts);
+ }
+
+ public static File asFile(String ... parts) {
+ return new File(asPath(parts));
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testSetup() throws Exception {
+ final int port = 8080;
+ final String topoId = "test_topology";
+ final String workerId = "worker_id";
+ final String user = "me";
+ final String stormLocal = asAbsPath("tmp", "testing");
+ final File workerArtifacts = asAbsFile(stormLocal, topoId, String.valueOf(port));
+ final File logMetadataFile = new File(workerArtifacts, "worker.yaml");
+ final File workerUserFile = asAbsFile(stormLocal, "workers-users", workerId);
+ final File workerRoot = asAbsFile(stormLocal, "workers", workerId);
+ final File distRoot = asAbsFile(stormLocal, "supervisor", "stormdist", topoId);
+
+ final Map<String, Object> topoConf = new HashMap<>();
+ final List<String> topoUsers = Arrays.asList("t-user-a", "t-user-b");
+ final List<String> logUsers = Arrays.asList("l-user-a", "l-user-b");
+
+ final List<String> topoGroups = Arrays.asList("t-group-a", "t-group-b");
+ final List<String> logGroups = Arrays.asList("l-group-a", "l-group-b");
+
+ topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, user);
+ topoConf.put(Config.LOGS_GROUPS, logGroups);
+ topoConf.put(Config.TOPOLOGY_GROUPS, topoGroups);
+ topoConf.put(Config.LOGS_USERS, logUsers);
+ topoConf.put(Config.TOPOLOGY_USERS, topoUsers);
+
+ final Map<String, Object> superConf = new HashMap<>();
+ superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
+ superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
+
+ final StringWriter yamlDump = new StringWriter();
+
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+ when(ops.fileExists(workerArtifacts)).thenReturn(true);
+ when(ops.fileExists(workerRoot)).thenReturn(true);
+ when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump);
+
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+ MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", 8080, la, workerId, topoConf, ops);
+
+ mc.setup();
+
+ //Initial Setup
+ verify(ops).forceMkdir(new File(workerRoot, "pids"));
+ verify(ops).forceMkdir(new File(workerRoot, "tmp"));
+ verify(ops).forceMkdir(new File(workerRoot, "heartbeats"));
+ verify(ops).fileExists(workerArtifacts);
+
+ //Log file permissions
+ verify(ops).getWriter(logMetadataFile);
+
+ String yamlResult = yamlDump.toString();
+ Yaml yaml = new Yaml();
+ Map<String, Object> result = (Map<String, Object>) yaml.load(yamlResult);
+ assertEquals(workerId, result.get("worker-id"));
+ assertEquals(user, result.get(Config.TOPOLOGY_SUBMITTER_USER));
+ HashSet<String> allowedUsers = new HashSet<>(topoUsers);
+ allowedUsers.addAll(logUsers);
+ assertEquals(allowedUsers, new HashSet<String>((List<String>)result.get(Config.LOGS_USERS)));
+
+ HashSet<String> allowedGroups = new HashSet<>(topoGroups);
+ allowedGroups.addAll(logGroups);
+ assertEquals(allowedGroups, new HashSet<String>((List<String>)result.get(Config.LOGS_GROUPS)));
+
+ //Save the current user to help with recovery
+ verify(ops).dump(workerUserFile, user);
+
+ //Create links to artifacts dir
+ verify(ops).createSymlink(new File(workerRoot, "artifacts"), workerArtifacts);
+
+ //Create links to blobs
+ verify(ops).createSymlink(new File(workerRoot, "resources"), new File(distRoot, "resources"));
+ }
+
+ @Test
+ public void testCleanup() throws Exception {
+ final int port = 8080;
+ final long pid = 100;
+ final String topoId = "test_topology";
+ final String workerId = "worker_id";
+ final String user = "me";
+ final String stormLocal = asAbsPath("tmp", "testing");
+ final File workerArtifacts = asAbsFile(stormLocal, topoId, String.valueOf(port));
+ final File logMetadataFile = new File(workerArtifacts, "worker.yaml");
+ final File workerUserFile = asAbsFile(stormLocal, "workers-users", workerId);
+ final File workerRoot = asAbsFile(stormLocal, "workers", workerId);
+ final File workerPidsRoot = new File(workerRoot, "pids");
+
+ final Map<String, Object> topoConf = new HashMap<>();
+ topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, user);
+
+ final Map<String, Object> superConf = new HashMap<>();
+ superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
+ superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
+
+ final StringWriter yamlDump = new StringWriter();
+
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+ when(ops.fileExists(workerArtifacts)).thenReturn(true);
+ when(ops.fileExists(workerRoot)).thenReturn(true);
+ when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump);
+
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+ MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", port, la, workerId, topoConf, ops);
+ mc.allPids.add(pid);
+
+ mc.cleanUp();
+ verify(ops).deleteIfExists(eq(new File(workerPidsRoot, String.valueOf(pid))), eq(user), any(String.class));
+
+ verify(ops).deleteIfExists(eq(new File(workerRoot, "pids")), eq(user), any(String.class));
+ verify(ops).deleteIfExists(eq(new File(workerRoot, "tmp")), eq(user), any(String.class));
+ verify(ops).deleteIfExists(eq(new File(workerRoot, "heartbeats")), eq(user), any(String.class));
+ verify(ops).deleteIfExists(eq(workerRoot), eq(user), any(String.class));
+ verify(ops).deleteIfExists(workerUserFile);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java
new file mode 100644
index 0000000..24ccda5
--- /dev/null
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java
@@ -0,0 +1,515 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.storm.daemon.supervisor.Slot.StaticState;
+import org.apache.storm.daemon.supervisor.Slot.TopoProfileAction;
+import org.apache.storm.daemon.supervisor.Slot.DynamicState;
+import org.apache.storm.daemon.supervisor.Slot.MachineState;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.NodeInfo;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.generated.WorkerResources;
+import org.apache.storm.localizer.ILocalizer;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Time;
+import org.junit.Test;
+
+public class SlotTest {
+ static WorkerResources mkWorkerResources(Double cpu, Double mem_on_heap, Double mem_off_heap) {
+ WorkerResources resources = new WorkerResources();
+ if (cpu != null) {
+ resources.set_cpu(cpu);
+ }
+
+ if (mem_on_heap != null) {
+ resources.set_mem_on_heap(mem_on_heap);
+ }
+
+ if (mem_off_heap != null) {
+ resources.set_mem_off_heap(mem_off_heap);
+ }
+ return resources;
+ }
+
+ static LSWorkerHeartbeat mkWorkerHB(String id, int port, List<ExecutorInfo> exec, Integer timeSecs) {
+ LSWorkerHeartbeat ret = new LSWorkerHeartbeat();
+ ret.set_topology_id(id);
+ ret.set_port(port);
+ ret.set_executors(exec);
+ ret.set_time_secs(timeSecs);
+ return ret;
+ }
+
+ static List<ExecutorInfo> mkExecutorInfoList(int ... executors) {
+ ArrayList<ExecutorInfo> ret = new ArrayList<>(executors.length);
+ for (int exec : executors) {
+ ExecutorInfo execInfo = new ExecutorInfo();
+ execInfo.set_task_start(exec);
+ execInfo.set_task_end(exec);
+ ret.add(execInfo);
+ }
+ return ret;
+ }
+
+ static LocalAssignment mkLocalAssignment(String id, List<ExecutorInfo> exec, WorkerResources resources) {
+ LocalAssignment ret = new LocalAssignment();
+ ret.set_topology_id(id);
+ ret.set_executors(exec);
+ if (resources != null) {
+ ret.set_resources(resources);
+ }
+ return ret;
+ }
+
+ @Test
+ public void testEquivilant() {
+ LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 100.0, 100.0));
+ LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 200.0, 100.0));
+ LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1,2,3,4,5,6), mkWorkerResources(100.0, 100.0, 100.0));
+ LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6,5,4,3,2,1), mkWorkerResources(100.0, 100.0, 100.0));
+
+ assertTrue(Slot.equivalent(null, null));
+ assertTrue(Slot.equivalent(a, a));
+ assertTrue(Slot.equivalent(b, bReordered));
+ assertTrue(Slot.equivalent(bReordered, b));
+
+ assertFalse(Slot.equivalent(a, aResized));
+ assertFalse(Slot.equivalent(aResized, a));
+ assertFalse(Slot.equivalent(a, null));
+ assertFalse(Slot.equivalent(null, b));
+ assertFalse(Slot.equivalent(a, b));
+ }
+
+ @Test
+ public void testEmptyToEmpty() throws Exception {
+ Time.startSimulatingAutoAdvanceOnSleep(1010);
+ try {
+ ILocalizer localizer = mock(ILocalizer.class);
+ LocalState state = mock(LocalState.class);
+ ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+ ISupervisor iSuper = mock(ISupervisor.class);
+ StaticState staticState = new StaticState(localizer, 1000, 1000, 1000, 1000,
+ containerLauncher, "localhost", 8080, iSuper, state);
+ DynamicState dynamicState = new DynamicState(null, null, null);
+ DynamicState nextState = Slot.handleEmpty(dynamicState, staticState);
+ assertEquals(MachineState.EMPTY, nextState.state);
+ assertTrue(Time.currentTimeMillis() > 1000);
+ } finally {
+ Time.stopSimulating();
+ }
+ }
+
+ @Test
+ public void testLaunchContainerFromEmpty() throws Exception {
+ Time.startSimulatingAutoAdvanceOnSleep(1010);
+ try {
+ int port = 8080;
+ String topoId = "NEW";
+ List<ExecutorInfo> execList = mkExecutorInfoList(1,2,3,4,5);
+ LocalAssignment newAssignment =
+ mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
+
+ ILocalizer localizer = mock(ILocalizer.class);
+ Container container = mock(Container.class);
+ LocalState state = mock(LocalState.class);
+ ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+ when(containerLauncher.launchContainer(port, newAssignment, state)).thenReturn(container);
+ LSWorkerHeartbeat hb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs());
+ when(container.readHeartbeat()).thenReturn(hb, hb);
+
+ @SuppressWarnings("unchecked")
+ Future<Void> baseFuture = mock(Future.class);
+ when(localizer.requestDownloadBaseTopologyBlobs(newAssignment, port)).thenReturn(baseFuture);
+
+ @SuppressWarnings("unchecked")
+ Future<Void> blobFuture = mock(Future.class);
+ when(localizer.requestDownloadTopologyBlobs(newAssignment, port)).thenReturn(blobFuture);
+
+ ISupervisor iSuper = mock(ISupervisor.class);
+ StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+ containerLauncher, "localhost", port, iSuper, state);
+ DynamicState dynamicState = new DynamicState(null, null, null)
+ .withNewAssignment(newAssignment);
+
+ DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+ verify(localizer).requestDownloadBaseTopologyBlobs(newAssignment, port);
+ assertEquals(MachineState.WAITING_FOR_BASIC_LOCALIZATION, nextState.state);
+ assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
+ assertEquals(newAssignment, nextState.pendingLocalization);
+ assertEquals(0, Time.currentTimeMillis());
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ verify(baseFuture).get(1000, TimeUnit.MILLISECONDS);
+ verify(localizer).requestDownloadTopologyBlobs(newAssignment, port);
+ assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
+ assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
+ assertEquals(newAssignment, nextState.pendingLocalization);
+ assertEquals(0, Time.currentTimeMillis());
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
+ verify(containerLauncher).launchContainer(port, newAssignment, state);
+ assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
+ assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+ assertSame(null, nextState.pendingLocalization);
+ assertSame(newAssignment, nextState.currentAssignment);
+ assertSame(container, nextState.container);
+ assertEquals(0, Time.currentTimeMillis());
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+ assertSame(null, nextState.pendingLocalization);
+ assertSame(newAssignment, nextState.currentAssignment);
+ assertSame(container, nextState.container);
+ assertEquals(0, Time.currentTimeMillis());
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+ assertSame(null, nextState.pendingLocalization);
+ assertSame(newAssignment, nextState.currentAssignment);
+ assertSame(container, nextState.container);
+ assertTrue(Time.currentTimeMillis() > 1000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+ assertSame(null, nextState.pendingLocalization);
+ assertSame(newAssignment, nextState.currentAssignment);
+ assertSame(container, nextState.container);
+ assertTrue(Time.currentTimeMillis() > 2000);
+ } finally {
+ Time.stopSimulating();
+ }
+ }
+
+
+ @Test
+ public void testRelaunch() throws Exception {
+ Time.startSimulatingAutoAdvanceOnSleep(1010);
+ try {
+ int port = 8080;
+ String topoId = "CURRENT";
+ List<ExecutorInfo> execList = mkExecutorInfoList(1,2,3,4,5);
+ LocalAssignment assignment =
+ mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
+
+ ILocalizer localizer = mock(ILocalizer.class);
+ Container container = mock(Container.class);
+ ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+ LSWorkerHeartbeat oldhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs()-10);
+ LSWorkerHeartbeat goodhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs());
+ when(container.readHeartbeat()).thenReturn(oldhb, oldhb, goodhb, goodhb);
+ when(container.areAllProcessesDead()).thenReturn(false, true);
+
+ ISupervisor iSuper = mock(ISupervisor.class);
+ LocalState state = mock(LocalState.class);
+ StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+ containerLauncher, "localhost", port, iSuper, state);
+ DynamicState dynamicState = new DynamicState(assignment, container, assignment);
+
+ DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+ assertEquals(MachineState.KILL_AND_RELAUNCH, nextState.state);
+ verify(container).kill();
+ assertTrue(Time.currentTimeMillis() > 1000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.KILL_AND_RELAUNCH, nextState.state);
+ verify(container).forceKill();
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
+ verify(container).relaunch();
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
+ assertTrue(Time.currentTimeMillis() > 3000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ } finally {
+ Time.stopSimulating();
+ }
+ }
+
+ @Test
+ public void testReschedule() throws Exception {
+ Time.startSimulatingAutoAdvanceOnSleep(1010);
+ try {
+ int port = 8080;
+ String cTopoId = "CURRENT";
+ List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
+ LocalAssignment cAssignment =
+ mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
+
+ Container cContainer = mock(Container.class);
+ LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
+ when(cContainer.readHeartbeat()).thenReturn(chb);
+ when(cContainer.areAllProcessesDead()).thenReturn(false, true);
+
+ String nTopoId = "NEW";
+ List<ExecutorInfo> nExecList = mkExecutorInfoList(1,2,3,4,5);
+ LocalAssignment nAssignment =
+ mkLocalAssignment(nTopoId, nExecList, mkWorkerResources(100.0, 100.0, 100.0));
+
+ ILocalizer localizer = mock(ILocalizer.class);
+ Container nContainer = mock(Container.class);
+ LocalState state = mock(LocalState.class);
+ ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+ when(containerLauncher.launchContainer(port, nAssignment, state)).thenReturn(nContainer);
+ LSWorkerHeartbeat nhb = mkWorkerHB(nTopoId, 100, nExecList, Time.currentTimeSecs());
+ when(nContainer.readHeartbeat()).thenReturn(nhb, nhb);
+
+ @SuppressWarnings("unchecked")
+ Future<Void> baseFuture = mock(Future.class);
+ when(localizer.requestDownloadBaseTopologyBlobs(nAssignment, port)).thenReturn(baseFuture);
+
+ @SuppressWarnings("unchecked")
+ Future<Void> blobFuture = mock(Future.class);
+ when(localizer.requestDownloadTopologyBlobs(nAssignment, port)).thenReturn(blobFuture);
+
+ ISupervisor iSuper = mock(ISupervisor.class);
+ StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+ containerLauncher, "localhost", port, iSuper, state);
+ DynamicState dynamicState = new DynamicState(cAssignment, cContainer, nAssignment);
+
+ DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+ assertEquals(MachineState.KILL, nextState.state);
+ verify(cContainer).kill();
+ verify(localizer).requestDownloadBaseTopologyBlobs(nAssignment, port);
+ assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
+ assertEquals(nAssignment, nextState.pendingLocalization);
+ assertTrue(Time.currentTimeMillis() > 1000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.KILL, nextState.state);
+ verify(cContainer).forceKill();
+ assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
+ assertEquals(nAssignment, nextState.pendingLocalization);
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.WAITING_FOR_BASIC_LOCALIZATION, nextState.state);
+ verify(cContainer).cleanUp();
+ verify(localizer).releaseSlotFor(cAssignment, port);
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
+ verify(baseFuture).get(1000, TimeUnit.MILLISECONDS);
+ verify(localizer).requestDownloadTopologyBlobs(nAssignment, port);
+ assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
+ assertEquals(nAssignment, nextState.pendingLocalization);
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
+ verify(containerLauncher).launchContainer(port, nAssignment, state);
+ assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
+ assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+ assertSame(null, nextState.pendingLocalization);
+ assertSame(nAssignment, nextState.currentAssignment);
+ assertSame(nContainer, nextState.container);
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+ assertSame(null, nextState.pendingLocalization);
+ assertSame(nAssignment, nextState.currentAssignment);
+ assertSame(nContainer, nextState.container);
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+ assertSame(null, nextState.pendingLocalization);
+ assertSame(nAssignment, nextState.currentAssignment);
+ assertSame(nContainer, nextState.container);
+ assertTrue(Time.currentTimeMillis() > 3000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+ assertSame(null, nextState.pendingLocalization);
+ assertSame(nAssignment, nextState.currentAssignment);
+ assertSame(nContainer, nextState.container);
+ assertTrue(Time.currentTimeMillis() > 4000);
+ } finally {
+ Time.stopSimulating();
+ }
+ }
+
+
+ @Test
+ public void testRunningToEmpty() throws Exception {
+ Time.startSimulatingAutoAdvanceOnSleep(1010);
+ try {
+ int port = 8080;
+ String cTopoId = "CURRENT";
+ List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
+ LocalAssignment cAssignment =
+ mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
+
+ Container cContainer = mock(Container.class);
+ LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
+ when(cContainer.readHeartbeat()).thenReturn(chb);
+ when(cContainer.areAllProcessesDead()).thenReturn(false, true);
+
+ ILocalizer localizer = mock(ILocalizer.class);
+ ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+
+ ISupervisor iSuper = mock(ISupervisor.class);
+ LocalState state = mock(LocalState.class);
+ StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+ containerLauncher, "localhost", port, iSuper, state);
+ DynamicState dynamicState = new DynamicState(cAssignment, cContainer, null);
+
+ DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+ assertEquals(MachineState.KILL, nextState.state);
+ verify(cContainer).kill();
+ verify(localizer, never()).requestDownloadBaseTopologyBlobs(null, port);
+ assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
+ assertEquals(null, nextState.pendingLocalization);
+ assertTrue(Time.currentTimeMillis() > 1000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.KILL, nextState.state);
+ verify(cContainer).forceKill();
+ assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
+ assertEquals(null, nextState.pendingLocalization);
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.EMPTY, nextState.state);
+ verify(cContainer).cleanUp();
+ verify(localizer).releaseSlotFor(cAssignment, port);
+ assertEquals(null, nextState.container);
+ assertEquals(null, nextState.currentAssignment);
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.EMPTY, nextState.state);
+ assertEquals(null, nextState.container);
+ assertEquals(null, nextState.currentAssignment);
+ assertTrue(Time.currentTimeMillis() > 3000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.EMPTY, nextState.state);
+ assertEquals(null, nextState.container);
+ assertEquals(null, nextState.currentAssignment);
+ assertTrue(Time.currentTimeMillis() > 3000);
+ } finally {
+ Time.stopSimulating();
+ }
+ }
+
+ @Test
+ public void testRunWithProfileActions() throws Exception {
+ Time.startSimulatingAutoAdvanceOnSleep(1010);
+ try {
+ int port = 8080;
+ String cTopoId = "CURRENT";
+ List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
+ LocalAssignment cAssignment =
+ mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
+
+ Container cContainer = mock(Container.class);
+ LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs()+100); //NOT going to timeout for a while
+ when(cContainer.readHeartbeat()).thenReturn(chb, chb, chb, chb, chb, chb);
+ when(cContainer.runProfiling(any(ProfileRequest.class), anyBoolean())).thenReturn(true);
+
+ ILocalizer localizer = mock(ILocalizer.class);
+ ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+
+ ISupervisor iSuper = mock(ISupervisor.class);
+ LocalState state = mock(LocalState.class);
+ StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+ containerLauncher, "localhost", port, iSuper, state);
+ Set<TopoProfileAction> profileActions = new HashSet<>();
+ ProfileRequest request = new ProfileRequest();
+ request.set_action(ProfileAction.JPROFILE_STOP);
+ NodeInfo info = new NodeInfo();
+ info.set_node("localhost");
+ info.add_to_port(port);
+ request.set_nodeInfo(info);
+ request.set_time_stamp(Time.currentTimeMillis() + 3000);//3 seconds from now
+
+ TopoProfileAction profile = new TopoProfileAction(cTopoId, request);
+ profileActions.add(profile);
+ Set<TopoProfileAction> expectedPending = new HashSet<>();
+ expectedPending.add(profile);
+
+
+ DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment).withProfileActions(profileActions, Collections.<TopoProfileAction> emptySet());
+
+ DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ verify(cContainer).runProfiling(request, false);
+ assertEquals(expectedPending, nextState.pendingStopProfileActions);
+ assertEquals(expectedPending, nextState.profileActions);
+ assertTrue(Time.currentTimeMillis() > 1000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertEquals(expectedPending, nextState.pendingStopProfileActions);
+ assertEquals(expectedPending, nextState.profileActions);
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertEquals(expectedPending, nextState.pendingStopProfileActions);
+ assertEquals(expectedPending, nextState.profileActions);
+ assertTrue(Time.currentTimeMillis() > 3000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ verify(cContainer).runProfiling(request, true);
+ assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.pendingStopProfileActions);
+ assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.profileActions);
+ assertTrue(Time.currentTimeMillis() > 4000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.pendingStopProfileActions);
+ assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.profileActions);
+ assertTrue(Time.currentTimeMillis() > 5000);
+ } finally {
+ Time.stopSimulating();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java b/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
new file mode 100644
index 0000000..41c17c8
--- /dev/null
+++ b/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.localizer;
+
+import static org.mockito.Matchers.*;
+import static org.mockito.Mockito.*;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.storm.daemon.supervisor.AdvancedFSOps;
+import org.apache.storm.utils.ConfigUtils;
+import org.junit.Test;
+
+import org.apache.storm.Config;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.localizer.LocalizedResource;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.security.auth.DefaultPrincipalToLocal;
+import org.apache.storm.utils.Utils;
+
+public class AsyncLocalizerTest {
+
+ @Test
+ public void testRequestDownloadBaseTopologyBlobs() throws Exception {
+ final String topoId = "TOPO";
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+ ExecutorInfo ei = new ExecutorInfo();
+ ei.set_task_start(1);
+ ei.set_task_end(1);
+ la.add_to_executors(ei);
+ final int port = 8080;
+ final String jarKey = topoId + "-stormjar.jar";
+ final String codeKey = topoId + "-stormcode.ser";
+ final String confKey = topoId + "-stormconf.ser";
+ final String stormLocal = "/tmp/storm-local/";
+ final String stormRoot = stormLocal+topoId+"/";
+ final File fStormRoot = new File(stormRoot);
+ ClientBlobStore blobStore = mock(ClientBlobStore.class);
+ Map<String, Object> conf = new HashMap<>();
+ conf.put(Config.SUPERVISOR_BLOBSTORE, ClientBlobStore.class.getName());
+ conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName());
+ conf.put(Config.STORM_CLUSTER_MODE, "distributed");
+ conf.put(Config.STORM_LOCAL_DIR, stormLocal);
+ Localizer localizer = mock(Localizer.class);
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ ConfigUtils mockedCU = mock(ConfigUtils.class);
+ Utils mockedU = mock(Utils.class);
+
+ Map<String, Object> topoConf = new HashMap<>(conf);
+
+ AsyncLocalizer al = new AsyncLocalizer(conf, localizer, ops);
+ ConfigUtils orig = ConfigUtils.setInstance(mockedCU);
+ Utils origUtils = Utils.setInstance(mockedU);
+ try {
+ when(mockedCU.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot);
+ when(mockedCU.supervisorLocalDirImpl(conf)).thenReturn(stormLocal);
+ when(mockedU.newInstanceImpl(ClientBlobStore.class)).thenReturn(blobStore);
+ when(mockedCU.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf);
+
+ Future<Void> f = al.requestDownloadBaseTopologyBlobs(la, port);
+ f.get(20, TimeUnit.SECONDS);
+ // We should be done now...
+
+ verify(blobStore).prepare(conf);
+ verify(mockedU).downloadResourcesAsSupervisorImpl(eq(jarKey), startsWith(stormLocal), eq(blobStore));
+ verify(mockedU).downloadResourcesAsSupervisorImpl(eq(codeKey), startsWith(stormLocal), eq(blobStore));
+ verify(mockedU).downloadResourcesAsSupervisorImpl(eq(confKey), startsWith(stormLocal), eq(blobStore));
+ verify(blobStore).shutdown();
+ //Extracting the dir from the jar
+ verify(mockedU).extractDirFromJarImpl(endsWith("stormjar.jar"), eq("resources"), any(File.class));
+ verify(ops).moveDirectoryPreferAtomic(any(File.class), eq(fStormRoot));
+ verify(ops).setupStormCodeDir(topoConf, fStormRoot);
+
+ verify(ops, never()).deleteIfExists(any(File.class));
+ } finally {
+ al.shutdown();
+ ConfigUtils.setInstance(orig);
+ Utils.setInstance(origUtils);
+ }
+ }
+
+ @Test
+ public void testRequestDownloadTopologyBlobs() throws Exception {
+ final String topoId = "TOPO-12345";
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+ ExecutorInfo ei = new ExecutorInfo();
+ ei.set_task_start(1);
+ ei.set_task_end(1);
+ la.add_to_executors(ei);
+ final String topoName = "TOPO";
+ final int port = 8080;
+ final String user = "user";
+ final String simpleLocalName = "simple.txt";
+ final String simpleKey = "simple";
+
+ final String stormLocal = "/tmp/storm-local/";
+ final File userDir = new File(stormLocal, user);
+ final String stormRoot = stormLocal+topoId+"/";
+
+ final String localizerRoot = "/tmp/storm-localizer/";
+ final String simpleLocalFile = localizerRoot + user + "/simple";
+ final String simpleCurrentLocalFile = localizerRoot + user + "/simple.current";
+
+ final StormTopology st = new StormTopology();
+ st.set_spouts(new HashMap<>());
+ st.set_bolts(new HashMap<>());
+ st.set_state_spouts(new HashMap<>());
+
+ Map<String, Map<String, Object>> topoBlobMap = new HashMap<>();
+ Map<String, Object> simple = new HashMap<>();
+ simple.put("localname", simpleLocalName);
+ simple.put("uncompress", false);
+ topoBlobMap.put(simpleKey, simple);
+
+ Map<String, Object> conf = new HashMap<>();
+ conf.put(Config.STORM_LOCAL_DIR, stormLocal);
+ Localizer localizer = mock(Localizer.class);
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ ConfigUtils mockedCU = mock(ConfigUtils.class);
+ Utils mockedU = mock(Utils.class);
+
+ Map<String, Object> topoConf = new HashMap<>(conf);
+ topoConf.put(Config.TOPOLOGY_BLOBSTORE_MAP, topoBlobMap);
+ topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, user);
+ topoConf.put(Config.TOPOLOGY_NAME, topoName);
+
+ List<LocalizedResource> localizedList = new ArrayList<>();
+ LocalizedResource simpleLocal = new LocalizedResource(simpleKey, simpleLocalFile, false);
+ localizedList.add(simpleLocal);
+
+ AsyncLocalizer al = new AsyncLocalizer(conf, localizer, ops);
+ ConfigUtils orig = ConfigUtils.setInstance(mockedCU);
+ Utils origUtils = Utils.setInstance(mockedU);
+ try {
+ when(mockedCU.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot);
+ when(mockedCU.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf);
+ when(mockedCU.readSupervisorTopologyImpl(conf, topoId, ops)).thenReturn(st);
+
+ when(localizer.getLocalUserFileCacheDir(user)).thenReturn(userDir);
+
+ when(localizer.getBlobs(any(List.class), eq(user), eq(topoName), eq(userDir))).thenReturn(localizedList);
+
+ Future<Void> f = al.requestDownloadTopologyBlobs(la, port);
+ f.get(20, TimeUnit.SECONDS);
+ // We should be done now...
+
+ verify(localizer).getLocalUserFileCacheDir(user);
+ verify(ops).fileExists(userDir);
+ verify(ops).forceMkdir(userDir);
+
+ verify(localizer).getBlobs(any(List.class), eq(user), eq(topoName), eq(userDir));
+
+ verify(ops).createSymlink(new File(stormRoot, simpleLocalName), new File(simpleCurrentLocalFile));
+ } finally {
+ al.shutdown();
+ ConfigUtils.setInstance(orig);
+ Utils.setInstance(origUtils);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java b/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
index ab2c9af..04b5ab2 100644
--- a/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
+++ b/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
@@ -34,8 +34,8 @@ public class LocalizedResourceRetentionSetTest {
// check adding reference to local resource with topology of same name
localresource2.addReference(("topo2"));
- lrset.addResource("key1", localresource1, false);
- lrset.addResource("key2", localresource2, false);
+ lrset.add("key1", localresource1, false);
+ lrset.add("key2", localresource2, false);
lrretset.addResources(lrset);
assertEquals("number to clean is not 0", 0, lrretset.getSizeWithNoReferences());
localresource1.removeReference(("topo1"));
@@ -64,9 +64,9 @@ public class LocalizedResourceRetentionSetTest {
// check adding reference to local resource with topology of same name
localresource2.addReference(("topo1"));
localresource2.setSize(10);
- lrset.addResource("key1", localresource1, false);
- lrset.addResource("key2", localresource2, false);
- lrset.addResource("archive1", archiveresource1, true);
+ lrset.add("key1", localresource1, false);
+ lrset.add("key2", localresource2, false);
+ lrset.add("archive1", archiveresource1, true);
lrretset.addResources(lrset);
assertEquals("number to clean is not 2", 2, lrretset.getSizeWithNoReferences());
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceSetTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceSetTest.java b/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceSetTest.java
index 7f20d19..550d695 100644
--- a/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceSetTest.java
+++ b/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceSetTest.java
@@ -37,9 +37,9 @@ public class LocalizedResourceSetTest {
LocalizedResource localresource1 = new LocalizedResource("key1", "testfile1", false, "topo1");
LocalizedResource localresource2 = new LocalizedResource("key2", "testfile2", true, "topo1");
assertEquals("size is wrong", 0, lrset.getSize());
- lrset.addResource("key1", localresource1, false);
+ lrset.add("key1", localresource1, false);
assertEquals("size is wrong", 1, lrset.getSize());
- lrset.addResource("key2", localresource2, true);
+ lrset.add("key2", localresource2, true);
assertEquals("size is wrong", 2, lrset.getSize());
}
@@ -48,8 +48,8 @@ public class LocalizedResourceSetTest {
LocalizedResourceSet lrset = new LocalizedResourceSet(user1);
LocalizedResource localresource1 = new LocalizedResource("key1", "testfile1", false, "topo1");
LocalizedResource localresource2 = new LocalizedResource("key2", "testfile2", true, "topo1");
- lrset.addResource("key1", localresource1, false);
- lrset.addResource("key2", localresource2, true);
+ lrset.add("key1", localresource1, false);
+ lrset.add("key2", localresource2, true);
assertEquals("get doesn't return same object", localresource1, lrset.get("key1", false));
assertEquals("get doesn't return same object", localresource2, lrset.get("key2", true));
@@ -60,8 +60,8 @@ public class LocalizedResourceSetTest {
LocalizedResourceSet lrset = new LocalizedResourceSet(user1);
LocalizedResource localresource1 = new LocalizedResource("key1", "testfile1", false, "topo1");
LocalizedResource localresource2 = new LocalizedResource("key2", "testfile2", true, "topo1");
- lrset.addResource("key1", localresource1, false);
- lrset.addResource("key2", localresource2, true);
+ lrset.add("key1", localresource1, false);
+ lrset.add("key2", localresource2, true);
assertEquals("doesn't exist", true, lrset.exists("key1", false));
assertEquals("doesn't exist", true, lrset.exists("key2", true));
boolean val = lrset.remove(localresource1);
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/localizer/LocalizerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/localizer/LocalizerTest.java b/storm-core/test/jvm/org/apache/storm/localizer/LocalizerTest.java
index 45ba108..613e165 100644
--- a/storm-core/test/jvm/org/apache/storm/localizer/LocalizerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/localizer/LocalizerTest.java
@@ -259,7 +259,7 @@ public class LocalizerTest {
// archive passed in must contain symlink named tmptestsymlink if not a zip file
public void testArchives(String archivePath, boolean supportSymlinks, int size) throws Exception {
- if (isOnWindows()) {
+ if (Utils.isOnWindows()) {
// Windows should set this to false cause symlink in compressed file doesn't work properly.
supportSymlinks = false;
}
@@ -669,11 +669,4 @@ public class LocalizerTest {
assertEquals("blob version not correct", 3, Utils.localVersionOfBlob(keyFile.toString()));
assertTrue("blob file with version 3 not created", new File(keyFile + ".3").exists());
}
-
- private boolean isOnWindows() {
- if (System.getenv("OS") != null) {
- return System.getenv("OS").equals("Windows_NT");
- }
- return false;
- }
}
[09/21] storm git commit: STORM-2018: Supervisor V2
Posted by ka...@apache.org.
STORM-2018: Supervisor V2
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/318ab5f3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/318ab5f3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/318ab5f3
Branch: refs/heads/1.x-branch
Commit: 318ab5f300e0820ab862cbeca04b7cb67699b938
Parents: 58cbfe5
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Sep 20 14:59:07 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Sep 20 16:54:14 2016 -0500
----------------------------------------------------------------------
bin/storm.py | 6 +-
log4j2/cluster.xml | 2 +-
pom.xml | 4 +-
.../cluster_state/zookeeper_state_factory.clj | 5 +-
.../org/apache/storm/command/kill_workers.clj | 33 -
.../clj/org/apache/storm/daemon/executor.clj | 4 +-
.../apache/storm/daemon/local_supervisor.clj | 28 +
.../clj/org/apache/storm/daemon/logviewer.clj | 7 +-
.../clj/org/apache/storm/daemon/supervisor.clj | 1272 ------------------
.../src/clj/org/apache/storm/daemon/worker.clj | 1 +
storm-core/src/clj/org/apache/storm/event.clj | 71 -
.../storm/pacemaker/pacemaker_state_factory.clj | 6 +-
storm-core/src/clj/org/apache/storm/testing.clj | 74 +-
.../jvm/org/apache/storm/ProcessSimulator.java | 82 ++
.../src/jvm/org/apache/storm/StormTimer.java | 242 ++++
.../storm/callback/DefaultWatcherCallBack.java | 35 +
.../apache/storm/callback/WatcherCallBack.java | 24 +
.../storm/callback/ZKStateChangedCallback.java | 25 +
.../storm/cluster/ClusterStateFactory.java | 4 +-
.../org/apache/storm/cluster/ClusterUtils.java | 244 ++++
.../org/apache/storm/cluster/ExecutorBeat.java | 44 +
.../org/apache/storm/cluster/IStateStorage.java | 219 +++
.../storm/cluster/IStormClusterState.java | 126 ++
.../storm/cluster/StateStorageFactory.java | 28 +
.../storm/cluster/StormClusterStateImpl.java | 720 ++++++++++
.../org/apache/storm/cluster/VersionedData.java | 36 +
.../apache/storm/cluster/ZKStateStorage.java | 252 ++++
.../org/apache/storm/command/HealthCheck.java | 126 ++
.../org/apache/storm/command/KillWorkers.java | 38 +
.../org/apache/storm/daemon/DaemonCommon.java | 22 +
.../storm/daemon/metrics/MetricsUtils.java | 19 +-
.../storm/daemon/supervisor/AdvancedFSOps.java | 335 +++++
.../storm/daemon/supervisor/BasicContainer.java | 638 +++++++++
.../supervisor/BasicContainerLauncher.java | 59 +
.../storm/daemon/supervisor/Container.java | 534 ++++++++
.../daemon/supervisor/ContainerLauncher.java | 96 ++
.../supervisor/ContainerRecoveryException.java | 29 +
.../DefaultUncaughtExceptionHandler.java | 31 +
.../supervisor/EventManagerPushCallback.java | 37 +
.../daemon/supervisor/ExitCodeCallback.java | 30 +
.../storm/daemon/supervisor/Killable.java | 50 +
.../storm/daemon/supervisor/LocalContainer.java | 85 ++
.../supervisor/LocalContainerLauncher.java | 60 +
.../daemon/supervisor/ReadClusterState.java | 327 +++++
.../daemon/supervisor/RunAsUserContainer.java | 96 ++
.../supervisor/RunAsUserContainerLauncher.java | 57 +
.../apache/storm/daemon/supervisor/Slot.java | 785 +++++++++++
.../daemon/supervisor/StandaloneSupervisor.java | 83 ++
.../storm/daemon/supervisor/Supervisor.java | 357 +++++
.../daemon/supervisor/SupervisorUtils.java | 311 +++++
.../supervisor/timer/SupervisorHealthCheck.java | 41 +
.../supervisor/timer/SupervisorHeartbeat.java | 87 ++
.../daemon/supervisor/timer/UpdateBlobs.java | 105 ++
.../org/apache/storm/event/EventManager.java | 24 +
.../org/apache/storm/event/EventManagerImp.java | 97 ++
.../apache/storm/localizer/AsyncLocalizer.java | 432 ++++++
.../org/apache/storm/localizer/ILocalizer.java | 70 +
.../localizer/LocalDownloadedResource.java | 146 ++
.../LocalizedResourceRetentionSet.java | 2 +-
.../storm/localizer/LocalizedResourceSet.java | 8 +-
.../org/apache/storm/localizer/Localizer.java | 38 +-
.../storm/metric/StormMetricsRegistry.java | 88 ++
.../org/apache/storm/nimbus/ILeaderElector.java | 6 +-
.../serialization/SerializationFactory.java | 17 +-
.../org/apache/storm/testing/FeederSpout.java | 27 +-
.../staticmocking/MockedSupervisorUtils.java | 31 +
.../apache/storm/trident/util/TridentUtils.java | 2 +-
.../jvm/org/apache/storm/utils/ConfigUtils.java | 536 ++++++++
.../org/apache/storm/utils/InprocMessaging.java | 69 +-
.../jvm/org/apache/storm/utils/LocalState.java | 112 +-
.../org/apache/storm/utils/NimbusClient.java | 19 +-
.../src/jvm/org/apache/storm/utils/Time.java | 51 +-
.../src/jvm/org/apache/storm/utils/Utils.java | 1054 +++++++++++++--
.../storm/zookeeper/LeaderElectorImp.java | 124 ++
.../apache/storm/zookeeper/ZkEventTypes.java | 43 +
.../apache/storm/zookeeper/ZkKeeperStates.java | 41 +
.../org/apache/storm/zookeeper/Zookeeper.java | 463 +++++++
.../org/apache/storm/integration_test.clj | 97 +-
.../org/apache/storm/testing4j_test.clj | 11 +-
.../clj/org/apache/storm/logviewer_test.clj | 55 +-
.../test/clj/org/apache/storm/metrics_test.clj | 9 +-
.../test/clj/org/apache/storm/nimbus_test.clj | 10 +-
.../storm/security/auth/drpc_auth_test.clj | 32 +-
.../clj/org/apache/storm/supervisor_test.clj | 751 -----------
.../daemon/supervisor/BasicContainerTest.java | 483 +++++++
.../storm/daemon/supervisor/ContainerTest.java | 265 ++++
.../storm/daemon/supervisor/SlotTest.java | 515 +++++++
.../storm/localizer/AsyncLocalizerTest.java | 187 +++
.../LocalizedResourceRetentionSetTest.java | 10 +-
.../localizer/LocalizedResourceSetTest.java | 12 +-
.../apache/storm/localizer/LocalizerTest.java | 9 +-
91 files changed, 11457 insertions(+), 2521 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/bin/storm.py
----------------------------------------------------------------------
diff --git a/bin/storm.py b/bin/storm.py
index 61aae98..e221780 100755
--- a/bin/storm.py
+++ b/bin/storm.py
@@ -553,7 +553,7 @@ def healthcheck(*args):
Run health checks on the local supervisor.
"""
exec_storm_class(
- "org.apache.storm.command.healthcheck",
+ "org.apache.storm.command.HealthCheck",
args=args,
jvmtype="-client",
extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR, "bin")])
@@ -566,7 +566,7 @@ def kill_workers(*args):
to have admin rights on the node to be able to successfully kill all workers.
"""
exec_storm_class(
- "org.apache.storm.command.kill_workers",
+ "org.apache.storm.command.KillWorkers",
args=args,
jvmtype="-client",
extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR, "bin")])
@@ -650,7 +650,7 @@ def pacemaker(klass="org.apache.storm.pacemaker.pacemaker"):
extrajars=cppaths,
jvmopts=jvmopts)
-def supervisor(klass="org.apache.storm.daemon.supervisor"):
+def supervisor(klass="org.apache.storm.daemon.supervisor.Supervisor"):
"""Syntax: [storm supervisor]
Launches the supervisor daemon. This command should be run
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/log4j2/cluster.xml
----------------------------------------------------------------------
diff --git a/log4j2/cluster.xml b/log4j2/cluster.xml
index e911823..a0ba3d1 100644
--- a/log4j2/cluster.xml
+++ b/log4j2/cluster.xml
@@ -18,7 +18,7 @@
<configuration monitorInterval="60">
<properties>
- <property name="pattern">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} [%p] %msg%n</property>
+ <property name="pattern">%d{yyyy-MM-dd HH:mm:ss.SSS} %t %c{1.} [%p] %msg%n</property>
</properties>
<appenders>
<RollingFile name="A1" immediateFlush="false"
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a1466c9..0665c10 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1067,8 +1067,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
- <source>1.7</source>
- <target>1.7</target>
+ <source>1.8</source>
+ <target>1.8</target>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/cluster_state/zookeeper_state_factory.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/cluster_state/zookeeper_state_factory.clj b/storm-core/src/clj/org/apache/storm/cluster_state/zookeeper_state_factory.clj
index 061c9e8..fdef972 100644
--- a/storm-core/src/clj/org/apache/storm/cluster_state/zookeeper_state_factory.clj
+++ b/storm-core/src/clj/org/apache/storm/cluster_state/zookeeper_state_factory.clj
@@ -17,13 +17,16 @@
(ns org.apache.storm.cluster-state.zookeeper-state-factory
(:import [org.apache.curator.framework.state ConnectionStateListener])
(:import [org.apache.zookeeper KeeperException$NoNodeException KeeperException$NodeExistsException]
- [org.apache.storm.cluster ClusterState DaemonType])
+ [org.apache.storm.cluster ClusterState DaemonType ZKStateStorage])
(:import [org.apache.storm.utils StormConnectionStateConverter])
(:use [org.apache.storm cluster config log util])
(:require [org.apache.storm [zookeeper :as zk]])
(:gen-class
:implements [org.apache.storm.cluster.ClusterStateFactory]))
+(defn -mkStore [this conf auth-conf, acls, context]
+ (ZKStateStorage. conf, auth-conf, acls, context))
+
(defn -mkState [this conf auth-conf acls context]
(let [zk (zk/mk-client conf (conf STORM-ZOOKEEPER-SERVERS) (conf STORM-ZOOKEEPER-PORT) :auth-conf auth-conf)]
(zk/mkdirs zk (conf STORM-ZOOKEEPER-ROOT) acls)
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/command/kill_workers.clj b/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
deleted file mode 100644
index 2670735..0000000
--- a/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
+++ /dev/null
@@ -1,33 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.command.kill-workers
- (:import [java.io File])
- (:use [org.apache.storm.daemon common])
- (:use [org.apache.storm util config])
- (:require [org.apache.storm.daemon
- [supervisor :as supervisor]])
- (:gen-class))
-
-(defn -main
- "Construct the supervisor-data from scratch and kill the workers on this supervisor"
- [& args]
- (let [conf (read-storm-config)
- conf (assoc conf STORM-LOCAL-DIR (. (File. (conf STORM-LOCAL-DIR)) getCanonicalPath))
- isupervisor (supervisor/standalone-supervisor)
- supervisor-data (supervisor/supervisor-data conf nil isupervisor)
- ids (supervisor/my-worker-ids conf)]
- (doseq [id ids]
- (supervisor/shutdown-worker supervisor-data id))))
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/daemon/executor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/executor.clj b/storm-core/src/clj/org/apache/storm/daemon/executor.clj
index 2436048..3b20b84 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/executor.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/executor.clj
@@ -208,9 +208,9 @@
(swap! interval-errors inc)
(when (<= @interval-errors max-per-interval)
- (cluster/report-error (:storm-cluster-state executor) (:storm-id executor) (:component-id executor)
+ (.report-error (:storm-cluster-state executor) (:storm-id executor) (:component-id executor)
(hostname storm-conf)
- (.getThisWorkerPort (:worker-context executor)) error)
+ (.getThisWorkerPort ^WorkerTopologyContext (:worker-context executor)) error)
))))
;; in its own function so that it can be mocked out by tracked topologies
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
new file mode 100644
index 0000000..a02a8e2
--- /dev/null
+++ b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
@@ -0,0 +1,28 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements. See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership. The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License. You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+(ns org.apache.storm.daemon.local-supervisor
+ (:import [org.apache.storm.daemon.supervisor Supervisor]
+ [org.apache.storm.utils ConfigUtils])
+ (:use [org.apache.storm.daemon common])
+ (:gen-class))
+
+(defserverfn mk-local-supervisor [conf shared-context isupervisor]
+ (if (not (ConfigUtils/isLocalMode conf))
+ (throw
+ (IllegalArgumentException. "Cannot start server in distrubuted mode!")))
+ (let [supervisor-server (Supervisor. conf shared-context isupervisor)]
+ (.launch supervisor-server)
+ supervisor-server))
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
index 1ad88f9..92224c6 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
@@ -33,11 +33,12 @@
(:import [java.nio ByteBuffer])
(:import [org.apache.storm.utils Utils])
(:import [org.apache.storm.daemon DirectoryCleaner])
+ (:import [org.apache.storm.daemon.supervisor SupervisorUtils])
(:import [org.yaml.snakeyaml Yaml]
[org.yaml.snakeyaml.constructor SafeConstructor])
(:import [org.apache.storm.ui InvalidRequestException]
[org.apache.storm.security.auth AuthUtils])
- (:require [org.apache.storm.daemon common [supervisor :as supervisor]])
+ (:require [org.apache.storm.daemon common])
(:require [compojure.route :as route]
[compojure.handler :as handler]
[ring.middleware.keyword-params]
@@ -157,10 +158,10 @@
(defn get-alive-ids
[conf now-secs]
(->>
- (supervisor/read-worker-heartbeats conf)
+ (clojurify-structure (SupervisorUtils/readWorkerHeartbeats conf))
(remove
#(or (not (val %))
- (supervisor/is-worker-hb-timed-out? now-secs
+ (SupervisorUtils/isWorkerHbTimedOut now-secs
(val %)
conf)))
keys
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
deleted file mode 100644
index 2888258..0000000
--- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
+++ /dev/null
@@ -1,1272 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.daemon.supervisor
- (:import [java.io File IOException FileOutputStream])
- (:import [org.apache.storm.scheduler ISupervisor]
- [org.apache.storm.utils LocalState Time Utils]
- [org.apache.storm.daemon Shutdownable]
- [org.apache.storm Constants]
- [org.apache.storm.cluster ClusterStateContext DaemonType]
- [java.net JarURLConnection]
- [java.net URI]
- [org.apache.commons.io FileUtils])
- (:use [org.apache.storm config util log timer local-state])
- (:import [org.apache.storm.generated AuthorizationException KeyNotFoundException WorkerResources])
- (:import [org.apache.storm.utils NimbusLeaderNotFoundException VersionInfo])
- (:import [java.nio.file Files StandardCopyOption])
- (:import [org.apache.storm Config])
- (:import [org.apache.storm.generated WorkerResources ProfileAction StormTopology])
- (:import [org.apache.storm.localizer LocalResource])
- (:use [org.apache.storm.daemon common])
- (:require [org.apache.storm.command [healthcheck :as healthcheck]])
- (:require [org.apache.storm.daemon [worker :as worker]]
- [org.apache.storm [process-simulator :as psim] [cluster :as cluster] [event :as event]]
- [clojure.set :as set])
- (:import [org.apache.thrift.transport TTransportException])
- (:import [org.apache.zookeeper data.ACL ZooDefs$Ids ZooDefs$Perms])
- (:import [org.yaml.snakeyaml Yaml]
- [org.yaml.snakeyaml.constructor SafeConstructor])
- (:require [metrics.gauges :refer [defgauge]])
- (:require [metrics.meters :refer [defmeter mark!]])
- (:gen-class
- :methods [^{:static true} [launch [org.apache.storm.scheduler.ISupervisor] void]]))
-
-; declare to avoid triggering forward reference issue
-(declare rm-topo-files assigned-storm-ids-from-port-assignments)
-
-(defmeter supervisor:num-workers-launched)
-
-(defmulti download-storm-code cluster-mode)
-(defmulti launch-worker (fn [supervisor & _] (cluster-mode (:conf supervisor))))
-
-(def STORM-VERSION (VersionInfo/getVersion))
-
-(defprotocol SupervisorDaemon
- (get-id [this])
- (get-conf [this])
- (shutdown-all-workers [this])
- )
-
-(defn- assignments-snapshot [storm-cluster-state callback assignment-versions]
- (let [storm-ids (.assignments storm-cluster-state callback)]
- (let [new-assignments
- (->>
- (dofor [sid storm-ids]
- (let [recorded-version (:version (get assignment-versions sid))]
- (if-let [assignment-version (.assignment-version storm-cluster-state sid callback)]
- (if (= assignment-version recorded-version)
- {sid (get assignment-versions sid)}
- {sid (.assignment-info-with-version storm-cluster-state sid callback)})
- {sid nil})))
- (apply merge)
- (filter-val not-nil?))
- new-profiler-actions
- (->>
- (dofor [sid (distinct storm-ids)]
- (if-let [topo-profile-actions (.get-topology-profile-requests storm-cluster-state sid false)]
- {sid topo-profile-actions}))
- (apply merge))]
-
- {:assignments (into {} (for [[k v] new-assignments] [k (:data v)]))
- :profiler-actions new-profiler-actions
- :versions new-assignments})))
-
-(defn- read-my-executors [assignments-snapshot storm-id assignment-id]
- (let [assignment (get assignments-snapshot storm-id)
- my-slots-resources (into {}
- (filter (fn [[[node _] _]] (= node assignment-id))
- (:worker->resources assignment)))
- my-executors (filter (fn [[_ [node _]]] (= node assignment-id))
- (:executor->node+port assignment))
- port-executors (apply merge-with
- concat
- (for [[executor [_ port]] my-executors]
- {port [executor]}
- ))]
- (into {} (for [[port executors] port-executors]
- ;; need to cast to int b/c it might be a long (due to how yaml parses things)
- ;; doall is to avoid serialization/deserialization problems with lazy seqs
- [(Integer. port) (mk-local-assignment storm-id (doall executors) (get my-slots-resources [assignment-id port]))]
- ))))
-
-(defn- read-assignments
- "Returns map from port to struct containing :storm-id, :executors and :resources"
- ([assignments-snapshot assignment-id]
- (->> (dofor [sid (keys assignments-snapshot)] (read-my-executors assignments-snapshot sid assignment-id))
- (apply merge-with (fn [& ignored] (throw-runtime "Should not have multiple topologies assigned to one port")))))
- ([assignments-snapshot assignment-id existing-assignment retries]
- (try (let [assignments (read-assignments assignments-snapshot assignment-id)]
- (reset! retries 0)
- assignments)
- (catch RuntimeException e
- (if (> @retries 2) (throw e) (swap! retries inc))
- (log-warn (.getMessage e) ": retrying " @retries " of 3")
- existing-assignment))))
-
-(defn- read-storm-code-locations
- [assignments-snapshot]
- (map-val :master-code-dir assignments-snapshot))
-
-(defn- read-downloaded-storm-ids [conf]
- (map #(url-decode %) (read-dir-contents (supervisor-stormdist-root conf)))
- )
-
-(defn read-worker-heartbeat [conf id]
- (let [local-state (worker-state conf id)]
- (try
- (ls-worker-heartbeat local-state)
- (catch Exception e
- (log-warn e "Failed to read local heartbeat for workerId : " id ",Ignoring exception.")
- nil))))
-
-
-(defn my-worker-ids [conf]
- (read-dir-contents (worker-root conf)))
-
-(defn read-worker-heartbeats
- "Returns map from worker id to heartbeat"
- [conf]
- (let [ids (my-worker-ids conf)]
- (into {}
- (dofor [id ids]
- [id (read-worker-heartbeat conf id)]))
- ))
-
-
-(defn matches-an-assignment? [worker-heartbeat assigned-executors]
- (let [local-assignment (assigned-executors (:port worker-heartbeat))]
- (and local-assignment
- (= (:storm-id worker-heartbeat) (:storm-id local-assignment))
- (= (disj (set (:executors worker-heartbeat)) Constants/SYSTEM_EXECUTOR_ID)
- (set (:executors local-assignment))))))
-
-(let [dead-workers (atom #{})]
- (defn get-dead-workers []
- @dead-workers)
- (defn add-dead-worker [worker]
- (swap! dead-workers conj worker))
- (defn remove-dead-worker [worker]
- (swap! dead-workers disj worker)))
-
-(defn is-worker-hb-timed-out? [now hb conf]
- (> (- now (:time-secs hb))
- (conf SUPERVISOR-WORKER-TIMEOUT-SECS)))
-
-(defn read-allocated-workers
- "Returns map from worker id to worker heartbeat. if the heartbeat is nil, then the worker is dead (timed out or never wrote heartbeat)"
- [supervisor assigned-executors now]
- (let [conf (:conf supervisor)
- ^LocalState local-state (:local-state supervisor)
- id->heartbeat (read-worker-heartbeats conf)
- approved-ids (set (keys (ls-approved-workers local-state)))]
- (into
- {}
- (dofor [[id hb] id->heartbeat]
- (let [state (cond
- (not hb)
- :not-started
- (or (not (contains? approved-ids id))
- (not (matches-an-assignment? hb assigned-executors)))
- :disallowed
- (or
- (when (get (get-dead-workers) id)
- (log-message "Worker Process " id " has died!")
- true)
- (is-worker-hb-timed-out? now hb conf))
- :timed-out
- true
- :valid)]
- (log-debug "Worker " id " is " state ": " (pr-str hb) " at supervisor time-secs " now)
- [id [state hb]]
- ))
- )))
-
-(defn- wait-for-worker-launch [conf id start-time]
- (let [state (worker-state conf id)]
- (loop []
- (let [hb (ls-worker-heartbeat state)]
- (when (and
- (not hb)
- (<
- (- (current-time-secs) start-time)
- (conf SUPERVISOR-WORKER-START-TIMEOUT-SECS)
- ))
- (log-message id " still hasn't started")
- (Time/sleep 500)
- (recur)
- )))
- (when-not (ls-worker-heartbeat state)
- (log-message "Worker " id " failed to start")
- )))
-
-(defn- wait-for-workers-launch [conf ids]
- (let [start-time (current-time-secs)]
- (doseq [id ids]
- (wait-for-worker-launch conf id start-time))
- ))
-
-(defn generate-supervisor-id []
- (uuid))
-
-(defnk worker-launcher [conf user args :environment {} :log-prefix nil :exit-code-callback nil :directory nil]
- (let [_ (when (clojure.string/blank? user)
- (throw (java.lang.IllegalArgumentException.
- "User cannot be blank when calling worker-launcher.")))
- wl-initial (conf SUPERVISOR-WORKER-LAUNCHER)
- storm-home (System/getProperty "storm.home")
- wl (if wl-initial wl-initial (str storm-home "/bin/worker-launcher"))
- command (concat [wl user] args)]
- (log-message "Running as user:" user " command:" (pr-str command))
- (launch-process command :environment environment :log-prefix log-prefix :exit-code-callback exit-code-callback :directory directory)
- ))
-
-(defnk worker-launcher-and-wait [conf user args :environment {} :log-prefix nil]
- (let [process (worker-launcher conf user args :environment environment)]
- (if log-prefix
- (read-and-log-stream log-prefix (.getInputStream process)))
- (try
- (.waitFor process)
- (catch InterruptedException e
- (log-message log-prefix " interrupted.")))
- (.exitValue process)))
-
-(defn- rmr-as-user
- "Launches a process owned by the given user that deletes the given path
- recursively. Throws RuntimeException if the directory is not removed."
- [conf id path]
- (let [user (Utils/getFileOwner path)]
- (worker-launcher-and-wait conf
- user
- ["rmr" path]
- :log-prefix (str "rmr " id))
- (if (exists-file? path)
- (throw (RuntimeException. (str path " was not deleted"))))))
-
-(defn try-cleanup-worker [conf id]
- (try
- (if (.exists (File. (worker-root conf id)))
- (do
- (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
- (rmr-as-user conf id (worker-root conf id))
- (do
- (rmr (worker-heartbeats-root conf id))
- ;; this avoids a race condition with worker or subprocess writing pid around same time
- (rmr (worker-pids-root conf id))
- (rmr (worker-root conf id))))
- (remove-worker-user! conf id)
- (remove-dead-worker id)
- ))
- (catch IOException e
- (log-warn-error e "Failed to cleanup worker " id ". Will retry later"))
- (catch RuntimeException e
- (log-warn-error e "Failed to cleanup worker " id ". Will retry later"))))
-
-(defn shutdown-worker [supervisor id]
- (log-message "Shutting down " (:supervisor-id supervisor) ":" id)
- (let [conf (:conf supervisor)
- pids (read-dir-contents (worker-pids-root conf id))
- thread-pid (@(:worker-thread-pids-atom supervisor) id)
- shutdown-sleep-secs (conf SUPERVISOR-WORKER-SHUTDOWN-SLEEP-SECS)
- as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
- user (get-worker-user conf id)]
- (when thread-pid
- (psim/kill-process thread-pid))
- (doseq [pid pids]
- (if as-user
- (worker-launcher-and-wait conf user ["signal" pid "15"] :log-prefix (str "kill -15 " pid))
- (kill-process-with-sig-term pid)))
- (when-not (empty? pids)
- (log-message "Sleep " shutdown-sleep-secs " seconds for execution of cleanup threads on worker.")
- (sleep-secs shutdown-sleep-secs))
- (doseq [pid pids]
- (if as-user
- (worker-launcher-and-wait conf user ["signal" pid "9"] :log-prefix (str "kill -9 " pid))
- (force-kill-process pid))
- (if as-user
- (try
- (rmr-as-user conf id (worker-pid-path conf id pid))
- (rmpath (worker-pid-path conf id pid))
- (rmpath (worker-tmp-root conf id pid))
- (catch Exception e
- (log-warn-error e "Failed to cleanup pid dir: " pid " for worker " id". Will retry later")))))
- ;; on windows, the supervisor may still holds the lock on the worker directory
- (try-cleanup-worker conf id))
- (log-message "Shut down " (:supervisor-id supervisor) ":" id))
-
-(def SUPERVISOR-ZK-ACLS
- [(first ZooDefs$Ids/CREATOR_ALL_ACL)
- (ACL. (bit-or ZooDefs$Perms/READ ZooDefs$Perms/CREATE) ZooDefs$Ids/ANYONE_ID_UNSAFE)])
-
-(defn supervisor-data [conf shared-context ^ISupervisor isupervisor]
- {:conf conf
- :shared-context shared-context
- :isupervisor isupervisor
- :active (atom true)
- :uptime (uptime-computer)
- :version STORM-VERSION
- :worker-thread-pids-atom (atom {})
- :storm-cluster-state (cluster/mk-storm-cluster-state conf :acls (when
- (Utils/isZkAuthenticationConfiguredStormServer
- conf)
- SUPERVISOR-ZK-ACLS)
- :context (ClusterStateContext. DaemonType/SUPERVISOR))
- :local-state (supervisor-state conf)
- :supervisor-id (.getSupervisorId isupervisor)
- :assignment-id (.getAssignmentId isupervisor)
- :my-hostname (hostname conf)
- :curr-assignment (atom nil) ;; used for reporting used ports when heartbeating
- :heartbeat-timer (mk-timer :kill-fn (fn [t]
- (log-error t "Error when processing event")
- (exit-process! 20 "Error when processing an event")
- ))
- :event-timer (mk-timer :kill-fn (fn [t]
- (log-error t "Error when processing event")
- (exit-process! 20 "Error when processing an event")
- ))
- :blob-update-timer (mk-timer :kill-fn (defn blob-update-timer
- [t]
- (log-error t "Error when processing event")
- (exit-process! 20 "Error when processing a event"))
- :timer-name "blob-update-timer")
- :localizer (Utils/createLocalizer conf (supervisor-local-dir conf))
- :assignment-versions (atom {})
- :sync-retry (atom 0)
- :download-lock (Object.)
- :stormid->profiler-actions (atom {})
- })
-
-(defn required-topo-files-exist?
- [conf storm-id]
- (let [stormroot (supervisor-stormdist-root conf storm-id)
- stormjarpath (supervisor-stormjar-path stormroot)
- stormcodepath (supervisor-stormcode-path stormroot)
- stormconfpath (supervisor-stormconf-path stormroot)]
- (and (every? exists-file? [stormroot stormconfpath stormcodepath])
- (or (local-mode? conf)
- (exists-file? stormjarpath)))))
-
-(defn get-worker-assignment-helper-msg
- [assignment supervisor port id]
- (str (pr-str assignment) " for this supervisor " (:supervisor-id supervisor) " on port "
- port " with id " id))
-
-(defn get-valid-new-worker-ids
- [conf supervisor reassign-executors new-worker-ids]
- (into {}
- (remove nil?
- (dofor [[port assignment] reassign-executors]
- (let [id (new-worker-ids port)
- storm-id (:storm-id assignment)
- ^WorkerResources resources (:resources assignment)
- mem-onheap (.get_mem_on_heap resources)]
- ;; This condition checks for required files exist before launching the worker
- (if (required-topo-files-exist? conf storm-id)
- (do
- (log-message "Launching worker with assignment "
- (get-worker-assignment-helper-msg assignment supervisor port id))
- (local-mkdirs (worker-pids-root conf id))
- (local-mkdirs (worker-tmp-root conf id))
- (local-mkdirs (worker-heartbeats-root conf id))
- (launch-worker supervisor
- (:storm-id assignment)
- port
- id
- mem-onheap)
- [id port])
- (do
- (log-message "Missing topology storm code, so can't launch worker with assignment "
- (get-worker-assignment-helper-msg assignment supervisor port id))
- nil)))))))
-
-(defn sync-processes [supervisor]
- (let [conf (:conf supervisor)
- ^LocalState local-state (:local-state supervisor)
- assigned-executors (defaulted (ls-local-assignments local-state) {})
- assigned-storm-ids (assigned-storm-ids-from-port-assignments assigned-executors)
- localizer (:localizer supervisor)
- now (current-time-secs)
- allocated (read-allocated-workers supervisor assigned-executors now)
- keepers (filter-val
- (fn [[state _]] (= state :valid))
- allocated)
- keep-ports (set (for [[id [_ hb]] keepers] (:port hb)))
- reassign-executors (select-keys-pred (complement keep-ports) assigned-executors)
- new-worker-ids (into
- {}
- (for [port (keys reassign-executors)]
- [port (uuid)]))
- all-downloaded-storm-ids (set (read-downloaded-storm-ids conf))]
- ;; 1. to kill are those in allocated that are dead or disallowed
- ;; 2. kill the ones that should be dead
- ;; - read pids, kill -9 and individually remove file
- ;; - rmr heartbeat dir, rmdir pid dir, rmdir id dir (catch exception and log)
- ;; 3. remove any downloaded code that's no longer assigned to this supervisor
- ;; 4. of the rest, figure out what assignments aren't yet satisfied
- ;; 5. generate new worker ids, write new "approved workers" to LS
- ;; 6. create local dir for worker id
- ;; 7. launch new workers (give worker-id, port, and supervisor-id)
- ;; 8. wait for workers launch
- ;; Note: sync-processes runs based on local assignment, and avoid reading zk assignment.
- ;; It's due to race condition between sync-supervisor and sync-processes.
-
- (log-debug "Syncing processes")
- (log-debug "Assigned executors: " assigned-executors)
- (log-debug "Allocated: " allocated)
- (doseq [[id [state heartbeat]] allocated]
- (when (not= :valid state)
- (log-message
- "Shutting down and clearing state for id " id
- ". Current supervisor time: " now
- ". State: " state
- ", Heartbeat: " (pr-str heartbeat))
- (shutdown-worker supervisor id)))
-
- (doseq [storm-id all-downloaded-storm-ids]
- (when-not (assigned-storm-ids storm-id)
- (log-message "Removing code for storm id "
- storm-id)
- (rm-topo-files conf storm-id localizer true)))
-
- (let [valid-new-worker-ids (get-valid-new-worker-ids conf supervisor reassign-executors new-worker-ids)]
- (ls-approved-workers! local-state
- (merge
- (select-keys (ls-approved-workers local-state)
- (keys keepers))
- valid-new-worker-ids))
- (wait-for-workers-launch conf (keys valid-new-worker-ids)))))
-
-(defn assigned-storm-ids-from-port-assignments [assignment]
- (->> assignment
- vals
- (map :storm-id)
- set))
-
-(defn get-blob-localname
- "Given the blob information either gets the localname field if it exists,
- else routines the default value passed in."
- [blob-info defaultValue]
- (or (get blob-info "localname") defaultValue))
-
-(defn should-uncompress-blob?
- "Given the blob information returns the value of the uncompress field, handling it either being
- a string or a boolean value, or if it's not specified then returns false"
- [blob-info]
- (Boolean. (get blob-info "uncompress")))
-
-(defn remove-blob-references
- "Remove a reference to a blob when its no longer needed."
- [localizer storm-id conf]
- (let [storm-conf (read-supervisor-storm-conf conf storm-id)
- blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)
- user (storm-conf TOPOLOGY-SUBMITTER-USER)
- topo-name (storm-conf TOPOLOGY-NAME)]
- (if blobstore-map
- (doseq [[k, v] blobstore-map]
- (.removeBlobReference localizer
- k
- user
- topo-name
- (should-uncompress-blob? v))))))
-
-(defn blobstore-map-to-localresources
- "Returns a list of LocalResources based on the blobstore-map passed in."
- [blobstore-map]
- (if blobstore-map
- (for [[k, v] blobstore-map] (LocalResource. k (should-uncompress-blob? v)))
- ()))
-
-(defn add-blob-references
- "For each of the downloaded topologies, adds references to the blobs that the topologies are
- using. This is used to reconstruct the cache on restart."
- [localizer storm-id conf]
- (let [storm-conf (read-supervisor-storm-conf conf storm-id)
- blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)
- user (storm-conf TOPOLOGY-SUBMITTER-USER)
- topo-name (storm-conf TOPOLOGY-NAME)
- localresources (blobstore-map-to-localresources blobstore-map)]
- (if blobstore-map
- (.addReferences localizer localresources user topo-name))))
-
-(defn rm-topo-files
- [conf storm-id localizer rm-blob-refs?]
- (let [path (supervisor-stormdist-root conf storm-id)]
- (try
- (if rm-blob-refs?
- (remove-blob-references localizer storm-id conf))
- (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
- (rmr-as-user conf storm-id path)
- (rmr (supervisor-stormdist-root conf storm-id)))
- (catch Exception e
- (log-message e (str "Exception removing: " storm-id))))))
-
-(defn verify-downloaded-files
- "Check for the files exists to avoid supervisor crashing
- Also makes sure there is no necessity for locking"
- [conf localizer assigned-storm-ids all-downloaded-storm-ids]
- (remove nil?
- (into #{}
- (for [storm-id all-downloaded-storm-ids
- :when (contains? assigned-storm-ids storm-id)]
- (when-not (required-topo-files-exist? conf storm-id)
- (log-debug "Files not present in topology directory")
- (rm-topo-files conf storm-id localizer false)
- storm-id)))))
-
-(defn mk-synchronize-supervisor [supervisor sync-processes event-manager processes-event-manager]
- (fn this []
- (let [conf (:conf supervisor)
- storm-cluster-state (:storm-cluster-state supervisor)
- ^ISupervisor isupervisor (:isupervisor supervisor)
- ^LocalState local-state (:local-state supervisor)
- sync-callback (fn [& ignored] (.add event-manager this))
- assignment-versions @(:assignment-versions supervisor)
- {assignments-snapshot :assignments
- storm-id->profiler-actions :profiler-actions
- versions :versions}
- (assignments-snapshot storm-cluster-state sync-callback assignment-versions)
- storm-code-map (read-storm-code-locations assignments-snapshot)
- all-downloaded-storm-ids (set (read-downloaded-storm-ids conf))
- existing-assignment (ls-local-assignments local-state)
- all-assignment (read-assignments assignments-snapshot
- (:assignment-id supervisor)
- existing-assignment
- (:sync-retry supervisor))
- new-assignment (->> all-assignment
- (filter-key #(.confirmAssigned isupervisor %)))
- assigned-storm-ids (assigned-storm-ids-from-port-assignments new-assignment)
- localizer (:localizer supervisor)
- checked-downloaded-storm-ids (set (verify-downloaded-files conf localizer assigned-storm-ids all-downloaded-storm-ids))
- downloaded-storm-ids (set/difference all-downloaded-storm-ids checked-downloaded-storm-ids)]
-
- ;; 1. download assigned topology code if it's not downloaded
- ;; 2. write new assignment to local assignment, and atom variables
- ;; 3. trigger sync-processes to take care of the rest
- ;; Note: mk-synchronize-supervisor runs based on zk assignment, and avoids touching local state
- ;; except updating new assignment which sync-processes will refer.
- ;; It's due to race condition between sync-supervisor and sync-processes.
-
- (log-debug "Synchronizing supervisor")
- (log-debug "Storm code map: " storm-code-map)
- (log-debug "All assignment: " all-assignment)
- (log-debug "New assignment: " new-assignment)
- (log-debug "Assigned Storm Ids " assigned-storm-ids)
- (log-debug "All Downloaded Ids " all-downloaded-storm-ids)
- (log-debug "Checked Downloaded Ids " checked-downloaded-storm-ids)
- (log-debug "Downloaded Ids " downloaded-storm-ids)
- (log-debug "Storm Ids Profiler Actions " storm-id->profiler-actions)
- ;; download code first
- ;; This might take awhile
- ;; - should this be done separately from usual monitoring?
- ;; should we only download when topology is assigned to this supervisor?
- (doseq [[storm-id master-code-dir] storm-code-map]
- (when (and (not (downloaded-storm-ids storm-id))
- (assigned-storm-ids storm-id))
- (log-message "Downloading code for storm id " storm-id)
- (try-cause
- (download-storm-code conf storm-id master-code-dir localizer)
-
- (catch NimbusLeaderNotFoundException e
- (log-warn-error e "Nimbus leader was not available."))
- (catch TTransportException e
- (log-warn-error e "There was a connection problem with nimbus.")))
- (log-message "Finished downloading code for storm id " storm-id)))
-
- (log-debug "Writing new assignment "
- (pr-str new-assignment))
- (doseq [p (set/difference (set (keys existing-assignment))
- (set (keys new-assignment)))]
- (.killedWorker isupervisor (int p)))
- (.assigned isupervisor (keys new-assignment))
- (ls-local-assignments! local-state
- new-assignment)
- (reset! (:assignment-versions supervisor) versions)
- (reset! (:stormid->profiler-actions supervisor) storm-id->profiler-actions)
- (reset! (:curr-assignment supervisor) new-assignment)
-
- (.add processes-event-manager sync-processes))))
-
-(defn mk-supervisor-capacities
- [conf]
- {Config/SUPERVISOR_MEMORY_CAPACITY_MB (double (conf SUPERVISOR-MEMORY-CAPACITY-MB))
- Config/SUPERVISOR_CPU_CAPACITY (double (conf SUPERVISOR-CPU-CAPACITY))})
-
-(defn update-blobs-for-topology!
- "Update each blob listed in the topology configuration if the latest version of the blob
- has not been downloaded."
- [conf storm-id localizer]
- (let [storm-conf (read-supervisor-storm-conf conf storm-id)
- blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)
- user (storm-conf TOPOLOGY-SUBMITTER-USER)
- localresources (blobstore-map-to-localresources blobstore-map)]
- (try
- (.updateBlobs localizer localresources user)
- (catch AuthorizationException authExp
- (log-error authExp))
- (catch KeyNotFoundException knf
- (log-error knf)))))
-
-(defn update-blobs-for-all-topologies-fn
- "Returns a function that downloads all blobs listed in the topology configuration for all topologies assigned
- to this supervisor, and creates version files with a suffix. The returned function is intended to be run periodically
- by a timer, created elsewhere."
- [supervisor]
- (fn []
- (try-cause
- (let [conf (:conf supervisor)
- downloaded-storm-ids (set (read-downloaded-storm-ids conf))
- new-assignment @(:curr-assignment supervisor)
- assigned-storm-ids (assigned-storm-ids-from-port-assignments new-assignment)]
- (doseq [topology-id downloaded-storm-ids]
- (let [storm-root (supervisor-stormdist-root conf topology-id)]
- (when (assigned-storm-ids topology-id)
- (log-debug "Checking Blob updates for storm topology id " topology-id " With target_dir: " storm-root)
- (update-blobs-for-topology! conf topology-id (:localizer supervisor))))))
- (catch TTransportException e
- (log-error
- e
- "Network error while updating blobs, will retry again later"))
- (catch NimbusLeaderNotFoundException e
- (log-error
- e
- "Nimbus unavailable to update blobs, will retry again later")))))
-
-(defn jvm-cmd [cmd]
- (let [java-home (.get (System/getenv) "JAVA_HOME")]
- (if (nil? java-home)
- cmd
- (str java-home file-path-separator "bin" file-path-separator cmd))))
-
-(defn java-cmd []
- (jvm-cmd "java"))
-
-(defn jmap-dump-cmd [profile-cmd pid target-dir]
- [profile-cmd pid "jmap" target-dir])
-
-(defn jstack-dump-cmd [profile-cmd pid target-dir]
- [profile-cmd pid "jstack" target-dir])
-
-(defn jprofile-start [profile-cmd pid]
- [profile-cmd pid "start"])
-
-(defn jprofile-stop [profile-cmd pid target-dir]
- [profile-cmd pid "stop" target-dir])
-
-(defn jprofile-dump [profile-cmd pid workers-artifacts-directory]
- [profile-cmd pid "dump" workers-artifacts-directory])
-
-(defn jprofile-jvm-restart [profile-cmd pid]
- [profile-cmd pid "kill"])
-
-(defn- delete-topology-profiler-action [storm-cluster-state storm-id profile-action]
- (log-message "Deleting profiler action.." profile-action)
- (.delete-topology-profile-requests storm-cluster-state storm-id profile-action))
-
-(defnk launch-profiler-action-for-worker
- "Launch profiler action for a worker"
- [conf user target-dir command :environment {} :exit-code-on-profile-action nil :log-prefix nil]
- (if-let [run-worker-as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)]
- (let [container-file (container-file-path target-dir)
- script-file (script-file-path target-dir)]
- (log-message "Running as user:" user " command:" (shell-cmd command))
- (if (exists-file? container-file) (rmr-as-user conf container-file container-file))
- (if (exists-file? script-file) (rmr-as-user conf script-file script-file))
- (worker-launcher
- conf
- user
- ["profiler" target-dir (write-script target-dir command :environment environment)]
- :log-prefix log-prefix
- :exit-code-callback exit-code-on-profile-action
- :directory (File. target-dir)))
- (launch-process
- command
- :environment environment
- :log-prefix log-prefix
- :exit-code-callback exit-code-on-profile-action
- :directory (File. target-dir))))
-
-(defn mk-run-profiler-actions-for-all-topologies
- "Returns a function that downloads all profile-actions listed for all topologies assigned
- to this supervisor, executes those actions as user and deletes them from zookeeper."
- [supervisor]
- (fn []
- (try
- (let [conf (:conf supervisor)
- stormid->profiler-actions @(:stormid->profiler-actions supervisor)
- storm-cluster-state (:storm-cluster-state supervisor)
- hostname (:my-hostname supervisor)
- storm-home (System/getProperty "storm.home")
- profile-cmd (str (clojure.java.io/file storm-home
- "bin"
- (conf WORKER-PROFILER-COMMAND)))]
- (doseq [[storm-id profiler-actions] stormid->profiler-actions]
- (when (not (empty? profiler-actions))
- (doseq [pro-action profiler-actions]
- (if (= hostname (:host pro-action))
- (let [port (:port pro-action)
- action ^ProfileAction (:action pro-action)
- stop? (> (System/currentTimeMillis) (:timestamp pro-action))
- target-dir (worker-artifacts-root conf storm-id port)
- storm-conf (read-supervisor-storm-conf conf storm-id)
- user (storm-conf TOPOLOGY-SUBMITTER-USER)
- environment (if-let [env (storm-conf TOPOLOGY-ENVIRONMENT)] env {})
- worker-pid (slurp (worker-artifacts-pid-path conf storm-id port))
- log-prefix (str "ProfilerAction process " storm-id ":" port " PROFILER_ACTION: " action " ")
- ;; Until PROFILER_STOP action is invalid, keep launching profiler start in case worker restarted
- ;; The profiler plugin script validates if JVM is recording before starting another recording.
- command (cond
- (= action ProfileAction/JMAP_DUMP) (jmap-dump-cmd profile-cmd worker-pid target-dir)
- (= action ProfileAction/JSTACK_DUMP) (jstack-dump-cmd profile-cmd worker-pid target-dir)
- (= action ProfileAction/JPROFILE_DUMP) (jprofile-dump profile-cmd worker-pid target-dir)
- (= action ProfileAction/JVM_RESTART) (jprofile-jvm-restart profile-cmd worker-pid)
- (and (not stop?)
- (= action ProfileAction/JPROFILE_STOP))
- (jprofile-start profile-cmd worker-pid) ;; Ensure the profiler is still running
- (and stop? (= action ProfileAction/JPROFILE_STOP)) (jprofile-stop profile-cmd worker-pid target-dir))
- action-on-exit (fn [exit-code]
- (log-message log-prefix " profile-action exited for code: " exit-code)
- (if stop?
- (delete-topology-profiler-action storm-cluster-state storm-id pro-action)))
- command (->> command (map str) (filter (complement empty?)))]
-
- (try
- (launch-profiler-action-for-worker conf
- user
- target-dir
- command
- :environment environment
- :exit-code-on-profile-action action-on-exit
- :log-prefix log-prefix)
- (catch IOException ioe
- (log-error ioe
- (str "Error in processing ProfilerAction '" action "' for " storm-id ":" port ", will retry later.")))
- (catch RuntimeException rte
- (log-error rte
- (str "Error in processing ProfilerAction '" action "' for " storm-id ":" port ", will retry later."))))))))))
- (catch Exception e
- (log-error e "Error running profiler actions, will retry again later")))))
-
-;; in local state, supervisor stores who its current assignments are
-;; another thread launches events to restart any dead processes if necessary
-(defserverfn mk-supervisor [conf shared-context ^ISupervisor isupervisor]
- (log-message "Starting Supervisor with conf " conf)
- (.prepare isupervisor conf (supervisor-isupervisor-dir conf))
- (FileUtils/cleanDirectory (File. (supervisor-tmp-dir conf)))
- (let [supervisor (supervisor-data conf shared-context isupervisor)
- [event-manager processes-event-manager :as managers] [(event/event-manager false) (event/event-manager false)]
- sync-processes (partial sync-processes supervisor)
- synchronize-supervisor (mk-synchronize-supervisor supervisor sync-processes event-manager processes-event-manager)
- synchronize-blobs-fn (update-blobs-for-all-topologies-fn supervisor)
- downloaded-storm-ids (set (read-downloaded-storm-ids conf))
- run-profiler-actions-fn (mk-run-profiler-actions-for-all-topologies supervisor)
- heartbeat-fn (fn [] (.supervisor-heartbeat!
- (:storm-cluster-state supervisor)
- (:supervisor-id supervisor)
- (->SupervisorInfo (current-time-secs)
- (:my-hostname supervisor)
- (:assignment-id supervisor)
- (keys @(:curr-assignment supervisor))
- ;; used ports
- (.getMetadata isupervisor)
- (conf SUPERVISOR-SCHEDULER-META)
- ((:uptime supervisor))
- (:version supervisor)
- (mk-supervisor-capacities conf))))]
- (heartbeat-fn)
-
- ;; should synchronize supervisor so it doesn't launch anything after being down (optimization)
- (schedule-recurring (:heartbeat-timer supervisor)
- 0
- (conf SUPERVISOR-HEARTBEAT-FREQUENCY-SECS)
- heartbeat-fn)
- (doseq [storm-id downloaded-storm-ids]
- (add-blob-references (:localizer supervisor) storm-id
- conf))
- ;; do this after adding the references so we don't try to clean things being used
- (.startCleaner (:localizer supervisor))
-
- (when (conf SUPERVISOR-ENABLE)
- ;; This isn't strictly necessary, but it doesn't hurt and ensures that the machine stays up
- ;; to date even if callbacks don't all work exactly right
- (schedule-recurring (:event-timer supervisor) 0 10 (fn [] (.add event-manager synchronize-supervisor)))
- (schedule-recurring (:event-timer supervisor)
- 0
- (conf SUPERVISOR-MONITOR-FREQUENCY-SECS)
- (fn [] (.add processes-event-manager sync-processes)))
-
- ;; Blob update thread. Starts with 30 seconds delay, every 30 seconds
- (schedule-recurring (:blob-update-timer supervisor)
- 30
- 30
- (fn [] (.add event-manager synchronize-blobs-fn)))
-
- (schedule-recurring (:event-timer supervisor)
- (* 60 5)
- (* 60 5)
- (fn [] (let [health-code (healthcheck/health-check conf)
- ids (my-worker-ids conf)]
- (if (not (= health-code 0))
- (do
- (doseq [id ids]
- (shutdown-worker supervisor id))
- (throw (RuntimeException. "Supervisor failed health check. Exiting.")))))))
-
- ;; Launch a thread that Runs profiler commands . Starts with 30 seconds delay, every 30 seconds
- (schedule-recurring (:event-timer supervisor)
- 30
- 30
- (fn [] (.add event-manager run-profiler-actions-fn))))
- (log-message "Starting supervisor with id " (:supervisor-id supervisor) " at host " (:my-hostname supervisor))
- (reify
- Shutdownable
- (shutdown [this]
- (log-message "Shutting down supervisor " (:supervisor-id supervisor))
- (reset! (:active supervisor) false)
- (cancel-timer (:heartbeat-timer supervisor))
- (cancel-timer (:event-timer supervisor))
- (cancel-timer (:blob-update-timer supervisor))
- (.shutdown event-manager)
- (.shutdown processes-event-manager)
- (.shutdown (:localizer supervisor))
- (.disconnect (:storm-cluster-state supervisor)))
- SupervisorDaemon
- (get-conf [this]
- conf)
- (get-id [this]
- (:supervisor-id supervisor))
- (shutdown-all-workers [this]
- (let [ids (my-worker-ids conf)]
- (doseq [id ids]
- (shutdown-worker supervisor id)
- )))
- DaemonCommon
- (waiting? [this]
- (or (not @(:active supervisor))
- (and
- (timer-waiting? (:heartbeat-timer supervisor))
- (timer-waiting? (:event-timer supervisor))
- (every? (memfn waiting?) managers)))
- ))))
-
-(defn kill-supervisor [supervisor]
- (.shutdown supervisor)
- )
-
-(defn setup-storm-code-dir
- [conf storm-conf dir]
- (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
- (worker-launcher-and-wait conf (storm-conf TOPOLOGY-SUBMITTER-USER) ["code-dir" dir] :log-prefix (str "setup conf for " dir))))
-
-(defn setup-blob-permission
- [conf storm-conf path]
- (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
- (worker-launcher-and-wait conf (storm-conf TOPOLOGY-SUBMITTER-USER) ["blob" path] :log-prefix (str "setup blob permissions for " path))))
-
-(defn download-blobs-for-topology!
- "Download all blobs passed by parameters for a given topology."
- [conf storm-conf localizer tmproot blobs fn-local-resources fn-symlink-name]
- (let [
- user (storm-conf TOPOLOGY-SUBMITTER-USER)
- topo-name (storm-conf TOPOLOGY-NAME)
- user-dir (.getLocalUserFileCacheDir localizer user)
- localresources (fn-local-resources blobs)]
- (when localresources
- (when-not (.exists user-dir)
- (FileUtils/forceMkdir user-dir))
- (try
- (let [localized-resources (.getBlobs localizer localresources user topo-name user-dir)]
- (setup-blob-permission conf storm-conf (.toString user-dir))
- (doseq [local-rsrc localized-resources]
- (let [rsrc-file-path (File. (.getFilePath local-rsrc))
- key-name (.getName rsrc-file-path)
- blob-symlink-target-name (.getName (File. (.getCurrentSymlinkPath local-rsrc)))
- symlink-name (fn-symlink-name blobs key-name)]
- (create-symlink! tmproot (.getParent rsrc-file-path) symlink-name
- blob-symlink-target-name))))
- (catch AuthorizationException authExp
- (log-error authExp))
- (catch KeyNotFoundException knf
- (log-error knf))))))
-
-(defn download-blobs-in-blobstore-map-for-topology!
- "Download all blobs listed in the topology configuration for a given topology."
- [conf stormconf-path localizer tmproot]
- (let [storm-conf (read-supervisor-storm-conf-given-path conf stormconf-path)
- blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)]
- (download-blobs-for-topology! conf storm-conf localizer tmproot blobstore-map
- (fn [blobs] (blobstore-map-to-localresources blobs))
- (fn [blobs key-name] (get-blob-localname (get blobs key-name) key-name)))))
-
-(defn download-dependencies-for-topology!
- "Download all dependencies blobs listed in the topology configuration for a given topology."
- [conf stormconf-path stormcode-path localizer tmproot]
- (let [storm-conf (read-supervisor-storm-conf-given-path conf stormconf-path)
- storm-code (read-supervisor-storm-code-given-path stormcode-path)
- dependencies (concat (.get_dependency_jars ^StormTopology storm-code)
- (.get_dependency_artifacts ^StormTopology storm-code))]
- (download-blobs-for-topology! conf storm-conf localizer tmproot dependencies
- (fn [blobs] (map #(LocalResource. % false) blobs))
- (fn [_ key-name] key-name))))
-
-(defn get-blob-file-names
- [blobstore-map]
- (if blobstore-map
- (for [[k, data] blobstore-map]
- (get-blob-localname data k))))
-
-(defn download-blobs-for-topology-succeed?
- "Assert if all blobs are downloaded for the given topology"
- [target-dir file-names]
- (if-not (empty? file-names)
- (every? #(Utils/checkFileExists target-dir %) file-names)
- true))
-
-(defn download-blobs-in-blobstore-map-for-topology-succeed?
- "Assert if all blobs in blobstore map are downloaded for the given topology"
- [stormconf-path target-dir]
- (let [storm-conf (clojurify-structure (Utils/fromCompressedJsonConf (FileUtils/readFileToByteArray (File. stormconf-path))))
- blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)
- file-names (get-blob-file-names blobstore-map)]
- (download-blobs-for-topology-succeed? target-dir file-names)))
-
-(defn download-dependencies-for-topology-succeed?
- "Assert if all dependencies blobs are downloaded for the given topology"
- [stormcode-path target-dir]
- (let [storm-code (read-supervisor-storm-code-given-path stormcode-path)
- file-names (concat (.get_dependency_jars ^StormTopology storm-code)
- (.get_dependency_artifacts ^StormTopology storm-code))]
- (download-blobs-for-topology-succeed? target-dir file-names)))
-
-;; distributed implementation
-(defmethod download-storm-code
- :distributed [conf storm-id master-code-dir localizer]
- ;; Downloading to permanent location is atomic
- (let [tmproot (str (supervisor-tmp-dir conf) file-path-separator (uuid))
- stormroot (supervisor-stormdist-root conf storm-id)
- blobstore (Utils/getClientBlobStoreForSupervisor conf)]
- (FileUtils/forceMkdir (File. tmproot))
- (if-not on-windows?
- (Utils/restrictPermissions tmproot)
- (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
- (throw-runtime (str "ERROR: Windows doesn't implement setting the correct permissions"))))
- (Utils/downloadResourcesAsSupervisor (master-stormjar-key storm-id)
- (supervisor-stormjar-path tmproot) blobstore)
- (Utils/downloadResourcesAsSupervisor (master-stormcode-key storm-id)
- (supervisor-stormcode-path tmproot) blobstore)
- (Utils/downloadResourcesAsSupervisor (master-stormconf-key storm-id)
- (supervisor-stormconf-path tmproot) blobstore)
- (.shutdown blobstore)
- (extract-dir-from-jar (supervisor-stormjar-path tmproot) RESOURCES-SUBDIR tmproot)
- (download-blobs-in-blobstore-map-for-topology! conf (supervisor-stormconf-path tmproot) localizer
- tmproot)
- (download-dependencies-for-topology! conf (supervisor-stormconf-path tmproot) (supervisor-stormcode-path tmproot) localizer tmproot)
- (if (and (download-blobs-in-blobstore-map-for-topology-succeed? (supervisor-stormconf-path tmproot) tmproot)
- (download-dependencies-for-topology-succeed? (supervisor-stormcode-path tmproot) tmproot))
- (do
- (log-message "Successfully downloaded blob resources for storm-id " storm-id)
- (if on-windows?
- ; Files/move with non-empty directory doesn't work well on Windows
- (FileUtils/moveDirectory (File. tmproot) (File. stormroot))
- (do
- (FileUtils/forceMkdir (File. stormroot))
- (Files/move (.toPath (File. tmproot)) (.toPath (File. stormroot))
- (doto (make-array StandardCopyOption 1) (aset 0 StandardCopyOption/ATOMIC_MOVE)))))
- (setup-storm-code-dir conf (read-supervisor-storm-conf conf storm-id) stormroot))
- (do
- (log-message "Failed to download blob resources for storm-id " storm-id)
- (rmr tmproot)))))
-
-(defn write-log-metadata-to-yaml-file! [storm-id port data conf]
- (let [file (get-log-metadata-file conf storm-id port)]
- ;;run worker as user needs the directory to have special permissions
- ;; or it is insecure
- (when (not (.exists (.getParentFile file)))
- (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
- (do (FileUtils/forceMkdir (.getParentFile file))
- (setup-storm-code-dir conf (read-supervisor-storm-conf conf storm-id) (.getCanonicalPath (.getParentFile file))))
- (.mkdirs (.getParentFile file))))
- (let [writer (java.io.FileWriter. file)
- yaml (Yaml.)]
- (try
- (.dump yaml data writer)
- (finally
- (.close writer))))))
-
-(defn write-log-metadata! [storm-conf user worker-id storm-id port conf]
- (let [data {TOPOLOGY-SUBMITTER-USER user
- "worker-id" worker-id
- LOGS-GROUPS (sort (distinct (remove nil?
- (concat
- (storm-conf LOGS-GROUPS)
- (storm-conf TOPOLOGY-GROUPS)))))
- LOGS-USERS (sort (distinct (remove nil?
- (concat
- (storm-conf LOGS-USERS)
- (storm-conf TOPOLOGY-USERS)))))}]
- (write-log-metadata-to-yaml-file! storm-id port data conf)))
-
-(defn jlp [stormroot conf]
- (let [resource-root (str stormroot File/separator RESOURCES-SUBDIR)
- os (clojure.string/replace (System/getProperty "os.name") #"\s+" "_")
- arch (System/getProperty "os.arch")
- arch-resource-root (str resource-root File/separator os "-" arch)]
- (str arch-resource-root File/pathSeparator resource-root File/pathSeparator (conf JAVA-LIBRARY-PATH))))
-
-(defn substitute-childopts
- "Generates runtime childopts by replacing keys with topology-id, worker-id, port, mem-onheap"
- [value worker-id topology-id port mem-onheap]
- (let [replacement-map {"%ID%" (str port)
- "%WORKER-ID%" (str worker-id)
- "%TOPOLOGY-ID%" (str topology-id)
- "%WORKER-PORT%" (str port)
- "%HEAP-MEM%" (str mem-onheap)}
- sub-fn #(reduce (fn [string entry]
- (apply clojure.string/replace string entry))
- %
- replacement-map)]
- (cond
- (nil? value) nil
- (sequential? value) (vec (map sub-fn value))
- :else (-> value sub-fn (clojure.string/split #"\s+")))))
-
-
-(defn create-blobstore-links
- "Create symlinks in worker launch directory for all blobs"
- [conf storm-id worker-id]
- (let [stormroot (supervisor-stormdist-root conf storm-id)
- storm-conf (read-supervisor-storm-conf conf storm-id)
- workerroot (worker-root conf worker-id)
- blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)
- blob-file-names (get-blob-file-names blobstore-map)
- resource-file-names (cons RESOURCES-SUBDIR blob-file-names)]
- (log-message "Creating symlinks for worker-id: " worker-id " storm-id: "
- storm-id " for files(" (count resource-file-names) "): " (pr-str resource-file-names))
- (create-symlink! workerroot stormroot RESOURCES-SUBDIR)
- (doseq [file-name blob-file-names]
- (create-symlink! workerroot stormroot file-name file-name))))
-
-(defn create-artifacts-link
- "Create a symlink from workder directory to its port artifacts directory"
- [conf storm-id port worker-id]
- (let [worker-dir (worker-root conf worker-id)
- topo-dir (worker-artifacts-root conf storm-id)]
- (log-message "Creating symlinks for worker-id: " worker-id " storm-id: "
- storm-id " to its port artifacts directory")
- (if (.exists (File. worker-dir))
- (create-symlink! worker-dir topo-dir "artifacts" port))))
-
-(defmethod launch-worker
- :distributed [supervisor storm-id port worker-id mem-onheap]
- (let [conf (:conf supervisor)
- run-worker-as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
- storm-home (System/getProperty "storm.home")
- storm-options (System/getProperty "storm.options")
- storm-conf-file (System/getProperty "storm.conf.file")
- worker-tmp-dir (worker-tmp-root conf worker-id)
- storm-log-dir LOG-DIR
- storm-log-conf-dir (conf STORM-LOG4J2-CONF-DIR)
- storm-log4j2-conf-dir (if storm-log-conf-dir
- (if (is-absolute-path? storm-log-conf-dir)
- storm-log-conf-dir
- (str storm-home file-path-separator storm-log-conf-dir))
- (str storm-home file-path-separator "log4j2"))
- stormroot (supervisor-stormdist-root conf storm-id)
- jlp (jlp stormroot conf)
- stormjar (supervisor-stormjar-path stormroot)
- storm-topology (read-supervisor-topology conf storm-id)
- dependencies (concat (.get_dependency_jars ^StormTopology storm-topology)
- (.get_dependency_artifacts ^StormTopology storm-topology))
- dependency-locations (map #(File. stormroot %) dependencies)
- storm-conf (read-supervisor-storm-conf conf storm-id)
- topo-classpath (if-let [cp (storm-conf TOPOLOGY-CLASSPATH)]
- [cp]
- [])
- classpath (-> (worker-classpath)
- (add-to-classpath [stormjar])
- (add-to-classpath dependency-locations)
- (add-to-classpath topo-classpath))
- top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS)
- mem-onheap (if (and mem-onheap (> mem-onheap 0)) ;; not nil and not zero
- (int (Math/ceil mem-onheap)) ;; round up
- (storm-conf WORKER-HEAP-MEMORY-MB)) ;; otherwise use default value
- gc-opts (substitute-childopts (if top-gc-opts top-gc-opts (conf WORKER-GC-CHILDOPTS)) worker-id storm-id port mem-onheap)
- topo-worker-logwriter-childopts (storm-conf TOPOLOGY-WORKER-LOGWRITER-CHILDOPTS)
- user (storm-conf TOPOLOGY-SUBMITTER-USER)
- logfilename "worker.log"
- workers-artifacts (worker-artifacts-root conf)
- logging-sensitivity (storm-conf TOPOLOGY-LOGGING-SENSITIVITY "S3")
- worker-childopts (when-let [s (conf WORKER-CHILDOPTS)]
- (substitute-childopts s worker-id storm-id port mem-onheap))
- topo-worker-childopts (when-let [s (storm-conf TOPOLOGY-WORKER-CHILDOPTS)]
- (substitute-childopts s worker-id storm-id port mem-onheap))
- worker--profiler-childopts (if (conf WORKER-PROFILER-ENABLED)
- (substitute-childopts (conf WORKER-PROFILER-CHILDOPTS) worker-id storm-id port mem-onheap)
- "")
- topology-worker-environment (if-let [env (storm-conf TOPOLOGY-ENVIRONMENT)]
- (merge env {"LD_LIBRARY_PATH" jlp})
- {"LD_LIBRARY_PATH" jlp})
-
- log4j-configuration-file (str (if (.startsWith (System/getProperty "os.name") "Windows")
- (if (.startsWith storm-log4j2-conf-dir "file:")
- storm-log4j2-conf-dir
- (str "file:///" storm-log4j2-conf-dir))
- storm-log4j2-conf-dir)
- file-path-separator "worker.xml")
- command (concat
- [(java-cmd) "-cp" classpath
- topo-worker-logwriter-childopts
- (str "-Dlogfile.name=" logfilename)
- (str "-Dstorm.home=" storm-home)
- (str "-Dworkers.artifacts=" workers-artifacts)
- (str "-Dstorm.id=" storm-id)
- (str "-Dworker.id=" worker-id)
- (str "-Dworker.port=" port)
- (str "-Dstorm.log.dir=" storm-log-dir)
- (str "-Dlog4j.configurationFile=" log4j-configuration-file)
- (str "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector")
- "org.apache.storm.LogWriter"]
- [(java-cmd) "-server"]
- worker-childopts
- topo-worker-childopts
- gc-opts
- worker--profiler-childopts
- [(str "-Djava.library.path=" jlp)
- (str "-Dlogfile.name=" logfilename)
- (str "-Dstorm.home=" storm-home)
- (str "-Dworkers.artifacts=" workers-artifacts)
- (str "-Dstorm.conf.file=" storm-conf-file)
- (str "-Dstorm.options=" storm-options)
- (str "-Dstorm.log.dir=" storm-log-dir)
- (str "-Djava.io.tmpdir=" worker-tmp-dir)
- (str "-Dlogging.sensitivity=" logging-sensitivity)
- (str "-Dlog4j.configurationFile=" log4j-configuration-file)
- (str "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector")
- (str "-Dstorm.id=" storm-id)
- (str "-Dworker.id=" worker-id)
- (str "-Dworker.port=" port)
- "-cp" classpath
- "org.apache.storm.daemon.worker"
- storm-id
- (:assignment-id supervisor)
- port
- worker-id])
- command (->> command (map str) (filter (complement empty?)))]
- (log-message "Launching worker with command: " (shell-cmd command))
- (write-log-metadata! storm-conf user worker-id storm-id port conf)
- (set-worker-user! conf worker-id user)
- (create-artifacts-link conf storm-id port worker-id)
- (let [log-prefix (str "Worker Process " worker-id)
- callback (fn [exit-code]
- (log-message log-prefix " exited with code: " exit-code)
- (add-dead-worker worker-id))
- worker-dir (worker-root conf worker-id)]
- (remove-dead-worker worker-id)
- (create-blobstore-links conf storm-id worker-id)
- (if run-worker-as-user
- (worker-launcher conf user ["worker" worker-dir (write-script worker-dir command :environment topology-worker-environment)] :log-prefix log-prefix :exit-code-callback callback :directory (File. worker-dir))
- (launch-process command :environment topology-worker-environment :log-prefix log-prefix :exit-code-callback callback :directory (File. worker-dir)))
- )))
-
-;; local implementation
-
-(defn resources-jar []
- (->> (.split (current-classpath) File/pathSeparator)
- (filter #(.endsWith % ".jar"))
- (filter #(zip-contains-dir? % RESOURCES-SUBDIR))
- first ))
-
-(defmethod download-storm-code
- :local [conf storm-id master-code-dir localizer]
- (let [tmproot (str (supervisor-tmp-dir conf) file-path-separator (uuid))
- stormroot (supervisor-stormdist-root conf storm-id)
- blob-store (Utils/getNimbusBlobStore conf master-code-dir nil)]
- (try
- (FileUtils/forceMkdir (File. tmproot))
- (with-open [fos-storm-code (FileOutputStream. (supervisor-stormcode-path tmproot))
- fos-storm-conf (FileOutputStream. (supervisor-stormconf-path tmproot))]
- (.readBlobTo blob-store (master-stormcode-key storm-id) fos-storm-code nil)
- (.readBlobTo blob-store (master-stormconf-key storm-id) fos-storm-conf nil))
- (finally
- (.shutdown blob-store)))
- (FileUtils/moveDirectory (File. tmproot) (File. stormroot))
- (setup-storm-code-dir conf (read-supervisor-storm-conf conf storm-id) stormroot)
- (let [classloader (.getContextClassLoader (Thread/currentThread))
- resources-jar (resources-jar)
- url (.getResource classloader RESOURCES-SUBDIR)
- target-dir (str stormroot file-path-separator RESOURCES-SUBDIR)]
- (cond
- resources-jar
- (do
- (log-message "Extracting resources from jar at " resources-jar " to " target-dir)
- (extract-dir-from-jar resources-jar RESOURCES-SUBDIR stormroot))
- url
- (do
- (log-message "Copying resources at " (str url) " to " target-dir)
- (FileUtils/copyDirectory (File. (.getFile url)) (File. target-dir)))))))
-
-(defmethod launch-worker
- :local [supervisor storm-id port worker-id mem-onheap]
- (let [conf (:conf supervisor)
- pid (uuid)
- worker (worker/mk-worker conf
- (:shared-context supervisor)
- storm-id
- (:assignment-id supervisor)
- port
- worker-id)]
- (set-worker-user! conf worker-id "")
- (psim/register-process pid worker)
- (swap! (:worker-thread-pids-atom supervisor) assoc worker-id pid)
- ))
-
-(defn -launch
- [supervisor]
- (log-message "Starting supervisor for storm version '" STORM-VERSION "'")
- (let [conf (read-storm-config)]
- (validate-distributed-mode! conf)
- (let [supervisor (mk-supervisor conf nil supervisor)]
- (add-shutdown-hook-with-force-kill-in-1-sec #(.shutdown supervisor)))
- (defgauge supervisor:num-slots-used-gauge #(count (my-worker-ids conf)))
- (start-metrics-reporters conf)))
-
-(defn standalone-supervisor []
- (let [conf-atom (atom nil)
- id-atom (atom nil)]
- (reify ISupervisor
- (prepare [this conf local-dir]
- (reset! conf-atom conf)
- (let [state (LocalState. local-dir)
- curr-id (if-let [id (ls-supervisor-id state)]
- id
- (generate-supervisor-id))]
- (ls-supervisor-id! state curr-id)
- (reset! id-atom curr-id))
- )
- (confirmAssigned [this port]
- true)
- (getMetadata [this]
- (doall (map int (get @conf-atom SUPERVISOR-SLOTS-PORTS))))
- (getSupervisorId [this]
- @id-atom)
- (getAssignmentId [this]
- @id-atom)
- (killedWorker [this port]
- )
- (assigned [this ports]
- ))))
-
-(defn -main []
- (setup-default-uncaught-exception-handler)
- (-launch (standalone-supervisor)))
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/daemon/worker.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/worker.clj b/storm-core/src/clj/org/apache/storm/daemon/worker.clj
index 1326672..62a36a2 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/worker.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/worker.clj
@@ -343,6 +343,7 @@
(let [q-metrics (.getMetrics queue)]
(/ (double (.population q-metrics)) (.capacity q-metrics))))
short-executor-receive-queue-map)
+ local-pop (map-key int local-pop)
remote-load (reduce merge (for [[np conn] @(:cached-node+port->socket worker)] (into {} (.getLoad conn remote-tasks))))
now (System/currentTimeMillis)]
(.setLocal load-mapping local-pop)
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/event.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/event.clj b/storm-core/src/clj/org/apache/storm/event.clj
deleted file mode 100644
index edc7616..0000000
--- a/storm-core/src/clj/org/apache/storm/event.clj
+++ /dev/null
@@ -1,71 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-
-(ns org.apache.storm.event
- (:use [org.apache.storm log util])
- (:import [org.apache.storm.utils Time Utils])
- (:import [java.io InterruptedIOException])
- (:import [java.util.concurrent LinkedBlockingQueue TimeUnit]))
-
-(defprotocol EventManager
- (add [this event-fn])
- (waiting? [this])
- (shutdown [this]))
-
-(defn event-manager
- "Creates a thread to respond to events. Any error will cause process to halt"
- [daemon?]
- (let [added (atom 0)
- processed (atom 0)
- ^LinkedBlockingQueue queue (LinkedBlockingQueue.)
- running (atom true)
- runner (Thread.
- (fn []
- (try-cause
- (while @running
- (let [r (.take queue)]
- (r)
- (swap! processed inc)))
- (catch InterruptedIOException t
- (log-message "Event manager interrupted while doing IO"))
- (catch InterruptedException t
- (log-message "Event manager interrupted"))
- (catch Throwable t
- (log-error t "Error when processing event")
- (exit-process! 20 "Error when processing an event")))))]
- (.setDaemon runner daemon?)
- (.start runner)
- (reify
- EventManager
-
- (add
- [this event-fn]
- ;; should keep track of total added and processed to know if this is finished yet
- (when-not @running
- (throw (RuntimeException. "Cannot add events to a shutdown event manager")))
- (swap! added inc)
- (.put queue event-fn))
-
- (waiting?
- [this]
- (or (Time/isThreadWaiting runner)
- (= @processed @added)))
-
- (shutdown
- [this]
- (reset! running false)
- (.interrupt runner)
- (.join runner)))))
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
index cede59e..077d528 100644
--- a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
+++ b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
@@ -26,7 +26,7 @@
HBExecutionException HBServerMessageType HBMessage
HBMessageData HBPulse]
[org.apache.storm.cluster_state zookeeper_state_factory]
- [org.apache.storm.cluster ClusterState]
+ [org.apache.storm.cluster ClusterState ZKStateStorage]
[org.apache.storm.pacemaker PacemakerClient])
(:gen-class
:implements [org.apache.storm.cluster.ClusterStateFactory]))
@@ -38,6 +38,10 @@
(defn makeZKState [conf auth-conf acls context]
(.mkState (zookeeper_state_factory.) conf auth-conf acls context))
+(defn -mkStore [this conf auth-conf, acls, context]
+ ;; This is only used by the supervisor (this does not support pacemaker)!!!
+ (ZKStateStorage. conf, auth-conf, acls, context))
+
(def max-retries 10)
(defn -mkState [this conf auth-conf acls context]
[15/21] storm git commit: Bad include for JDK 7 support
Posted by ka...@apache.org.
Bad include for JDK 7 support
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/905c4436
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/905c4436
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/905c4436
Branch: refs/heads/1.x-branch
Commit: 905c443653f579cd0af96d1c799843a1b718801f
Parents: b12c288
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Sep 26 09:21:32 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Sep 26 09:21:32 2016 -0500
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/905c4436/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
index f1140f7..977cb2b 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -30,7 +30,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.Config;
[14/21] storm git commit: Merge branch 'STORM-2122' of
https://github.com/revans2/incubator-storm into STORM-2122
Posted by ka...@apache.org.
Merge branch 'STORM-2122' of https://github.com/revans2/incubator-storm into STORM-2122
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b12c288d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b12c288d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b12c288d
Branch: refs/heads/1.x-branch
Commit: b12c288dcf04c3a89d8759fa56d836d8371d20cf
Parents: 2297757
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Sun Sep 25 20:57:04 2016 +0900
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Sep 26 08:23:36 2016 -0500
----------------------------------------------------------------------
.../storm/daemon/supervisor/BasicContainer.java | 93 +++++++++++++++++---
1 file changed, 79 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b12c288d/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
index b1fa226..f1140f7 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -23,8 +23,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.Iterator;
import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -470,6 +471,82 @@ public class BasicContainer extends Container {
return log4jConfigurationDir + Utils.FILE_PATH_SEPARATOR + "worker.xml";
}
+ private static class DependencyLocations {
+ private List<String> _data = null;
+ private final Map<String, Object> _conf;
+ private final String _topologyId;
+ private final AdvancedFSOps _ops;
+ private final String _stormRoot;
+
+ public DependencyLocations(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, final String stormRoot) {
+ _conf = conf;
+ _topologyId = topologyId;
+ _ops = ops;
+ _stormRoot = stormRoot;
+ }
+
+ public String toString() {
+ List<String> data;
+ synchronized(this) {
+ data = _data;
+ }
+ return "DEP_LOCS for " + _topologyId +" => " + data;
+ }
+
+ public synchronized List<String> get() throws IOException {
+ if (_data != null) {
+ return _data;
+ }
+ final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _ops);
+ final List<String> dependencyLocations = new ArrayList<>();
+ if (stormTopology.get_dependency_jars() != null) {
+ for (String dependency : stormTopology.get_dependency_jars()) {
+ dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath());
+ }
+ }
+
+ if (stormTopology.get_dependency_artifacts() != null) {
+ for (String dependency : stormTopology.get_dependency_artifacts()) {
+ dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath());
+ }
+ }
+ _data = dependencyLocations;
+ return _data;
+ }
+ }
+
+ static class DepLRUCache {
+ public final int _maxSize = 100; //We could make this configurable in the future...
+
+ @SuppressWarnings("serial")
+ private LinkedHashMap<String, DependencyLocations> _cache = new LinkedHashMap<String, DependencyLocations>() {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<String,DependencyLocations> eldest) {
+ return (size() > _maxSize);
+ }
+ };
+
+ public synchronized DependencyLocations get(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, String stormRoot) {
+ //Only go off of the topology id for now.
+ DependencyLocations dl = _cache.get(topologyId);
+ if (dl == null) {
+ _cache.putIfAbsent(topologyId, new DependencyLocations(conf, topologyId, ops, stormRoot));
+ dl = _cache.get(topologyId);
+ }
+ return dl;
+ }
+
+ public synchronized void clear() {
+ _cache.clear();
+ }
+ }
+
+ static final DepLRUCache DEP_LOC_CACHE = new DepLRUCache();
+
+ public static List<String> getDependencyLocationsFor(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, String stormRoot) throws IOException {
+ return DEP_LOC_CACHE.get(conf, topologyId, ops, stormRoot).get();
+ }
+
/**
* Get parameters for the class path of the worker process. Also used by the
* log Writer
@@ -479,19 +556,7 @@ public class BasicContainer extends Container {
*/
private List<String> getClassPathParams(final String stormRoot) throws IOException {
final String stormJar = ConfigUtils.supervisorStormJarPath(stormRoot);
- final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _ops);
- final List<String> dependencyLocations = new ArrayList<>();
- if (stormTopology.get_dependency_jars() != null) {
- for (String dependency : stormTopology.get_dependency_jars()) {
- dependencyLocations.add(new File(stormRoot, dependency).getAbsolutePath());
- }
- }
-
- if (stormTopology.get_dependency_artifacts() != null) {
- for (String dependency : stormTopology.get_dependency_artifacts()) {
- dependencyLocations.add(new File(stormRoot, dependency).getAbsolutePath());
- }
- }
+ final List<String> dependencyLocations = getDependencyLocationsFor(_conf, _topologyId, _ops, stormRoot);
final String workerClassPath = getWorkerClassPath(stormJar, dependencyLocations);
List<String> classPathParams = new ArrayList<>();
[06/21] storm git commit: STORM-2018: Supervisor V2
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerLauncher.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerLauncher.java
new file mode 100644
index 0000000..bfba2ee
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerLauncher.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.storm.Config;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.messaging.IContext;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Launches containers
+ */
+public abstract class ContainerLauncher {
+ private static final Logger LOG = LoggerFactory.getLogger(ContainerLauncher.class);
+
+ /**
+ * Factory to create the right container launcher
+ * for the config and the environment.
+ * @param conf the config
+ * @param supervisorId the ID of the supervisor
+ * @param sharedContext Used in local mode to let workers talk together without netty
+ * @return the proper container launcher
+ * @throws IOException on any error
+ */
+ public static ContainerLauncher make(Map<String, Object> conf, String supervisorId, IContext sharedContext) throws IOException {
+ if (ConfigUtils.isLocalMode(conf)) {
+ return new LocalContainerLauncher(conf, supervisorId, sharedContext);
+ }
+
+ if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
+ return new RunAsUserContainerLauncher(conf, supervisorId);
+ }
+ return new BasicContainerLauncher(conf, supervisorId);
+ }
+
+ protected ContainerLauncher() {
+ //Empty
+ }
+
+ /**
+ * Launch a container in a given slot
+ * @param port the port to run this on
+ * @param assignment what to launch
+ * @param state the current state of the supervisor
+ * @return The container that can be used to manager the processes.
+ * @throws IOException on any error
+ */
+ public abstract Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException;
+
+ /**
+ * Recover a container for a running process
+ * @param port the port the assignment is running on
+ * @param assignment the assignment that was launched
+ * @param state the current state of the supervisor
+ * @return The container that can be used to manage the processes.
+ * @throws IOException on any error
+ * @throws ContainerRecoveryException if the Container could not be recovered
+ */
+ public abstract Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException, ContainerRecoveryException;
+
+ /**
+ * Try to recover a container using just the worker ID.
+ * The result is really only useful for killing the container
+ * and so is returning a Killable. Even if a Container is returned
+ * do not case the result to Container because only the Killable APIs
+ * are guaranteed to work.
+ * @param workerId the id of the worker to use
+ * @param localState the state of the running supervisor
+ * @return a Killable that can be used to kill the underlying container.
+ * @throws IOException on any error
+ * @throws ContainerRecoveryException if the Container could not be recovered
+ */
+ public abstract Killable recoverContainer(String workerId, LocalState localState) throws IOException, ContainerRecoveryException;
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java
new file mode 100644
index 0000000..7ab6e67
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+/**
+ * Could not recover the container.
+ */
+public class ContainerRecoveryException extends RuntimeException {
+
+ public ContainerRecoveryException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
new file mode 100644
index 0000000..8785f86
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DefaultUncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(DefaultUncaughtExceptionHandler.class);
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ LOG.error("Error when processing event", e);
+ Utils.exitProcess(20, "Error when processing an event");
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java
new file mode 100644
index 0000000..177bf67
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.storm.event.EventManager;
+
+public class EventManagerPushCallback implements Runnable {
+
+ private EventManager eventManager;
+
+ private Runnable cb;
+
+ public EventManagerPushCallback(Runnable cb, EventManager eventManager) {
+ this.eventManager = eventManager;
+ this.cb = cb;
+ }
+
+ @Override
+ public void run() {
+ eventManager.add(cb);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java
new file mode 100644
index 0000000..082f205
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+/**
+ * A callback that can accept an integer.
+ */
+public interface ExitCodeCallback {
+
+ /**
+ * The process finished
+ * @param exitCode the exit code of the finished process.
+ */
+ public void call(int exitCode);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Killable.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Killable.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Killable.java
new file mode 100644
index 0000000..8d6d8e0
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Killable.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+
+public interface Killable {
+
+ /**
+ * Kill the processes in this container nicely.
+ * kill -15 equivalent
+ * @throws IOException on any error
+ */
+ public void kill() throws IOException;
+
+ /**
+ * Kill the processes in this container violently.
+ * kill -9 equivalent
+ * @throws IOException on any error
+ */
+ public void forceKill() throws IOException;
+
+ /**
+ * @return true if all of the processes are dead, else false
+ * @throws IOException on any error
+ */
+ public boolean areAllProcessesDead() throws IOException;
+
+ /**
+ * Clean up the container. It is not coming back.
+ * by default do the same thing as when restarting.
+ * @throws IOException on any error
+ */
+ public void cleanUp() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java
new file mode 100644
index 0000000..f21efa4
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.storm.ProcessSimulator;
+import org.apache.storm.daemon.Shutdownable;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.messaging.IContext;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import clojure.java.api.Clojure;
+import clojure.lang.IFn;
+
+public class LocalContainer extends Container {
+ private static final Logger LOG = LoggerFactory.getLogger(LocalContainer.class);
+ private volatile boolean _isAlive = false;
+ private final IContext _sharedContext;
+
+ public LocalContainer(Map<String, Object> conf, String supervisorId, int port, LocalAssignment assignment, IContext sharedContext) throws IOException {
+ super(ContainerType.LAUNCH, conf, supervisorId, port, assignment, null, null, null);
+ _sharedContext = sharedContext;
+ _workerId = Utils.uuid();
+ }
+
+ @Override
+ public void launch() throws IOException {
+ //TODO when worker goes to java, just call it directly (not through clojure)
+ IFn mkWorker = Clojure.var("org.apache.storm.daemon.worker", "mk-worker");
+
+ Shutdownable worker = (Shutdownable) mkWorker.invoke(_conf, _sharedContext, _topologyId, _supervisorId, _port, _workerId);
+ saveWorkerUser(System.getProperty("user.name"));
+ ProcessSimulator.registerProcess(_workerId, worker);
+ _isAlive = true;
+ }
+
+ @Override
+ public void kill() throws IOException {
+ ProcessSimulator.killProcess(_workerId);
+ _isAlive = false;
+ //Make sure the worker is down before we try to shoot any child processes
+ super.kill();
+ }
+
+ @Override
+ public boolean areAllProcessesDead() throws IOException {
+ return !_isAlive && super.areAllProcessesDead();
+ }
+
+ @Override
+ public void relaunch() throws IOException {
+ LOG.warn("NOOP relaunch in local mode...");
+ }
+
+ @Override
+ public boolean didMainProcessExit() {
+ //In local mode the main process should never exit on it's own
+ return false;
+ }
+
+ @Override
+ public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException {
+ throw new RuntimeException("Profiling requests are not supported in local mode");
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java
new file mode 100644
index 0000000..c25bc49
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.messaging.IContext;
+import org.apache.storm.utils.LocalState;
+
+/**
+ * Launch Containers in local mode.
+ */
+public class LocalContainerLauncher extends ContainerLauncher {
+ private final Map<String, Object> _conf;
+ private final String _supervisorId;
+ private final IContext _sharedContext;
+
+ public LocalContainerLauncher(Map<String, Object> conf, String supervisorId, IContext sharedContext) {
+ _conf = conf;
+ _supervisorId = supervisorId;
+ _sharedContext = sharedContext;
+ }
+
+ @Override
+ public Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+ LocalContainer ret = new LocalContainer(_conf, _supervisorId, port, assignment, _sharedContext);
+ ret.setup();
+ ret.launch();
+ return ret;
+ }
+
+ @Override
+ public Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+ //We are in the same process we cannot recover anything
+ throw new ContainerRecoveryException("Local Mode Recovery is not supported");
+ }
+
+ @Override
+ public Killable recoverContainer(String workerId, LocalState localState) throws IOException {
+ //We are in the same process we cannot recover anything
+ throw new ContainerRecoveryException("Local Mode Recovery is not supported");
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
new file mode 100644
index 0000000..27f18b6
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
@@ -0,0 +1,327 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.storm.Config;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.cluster.VersionedData;
+import org.apache.storm.daemon.supervisor.Slot.MachineState;
+import org.apache.storm.daemon.supervisor.Slot.TopoProfileAction;
+import org.apache.storm.event.EventManager;
+import org.apache.storm.generated.Assignment;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.NodeInfo;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.generated.WorkerResources;
+import org.apache.storm.localizer.ILocalizer;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ReadClusterState implements Runnable, AutoCloseable {
+ private static final Logger LOG = LoggerFactory.getLogger(ReadClusterState.class);
+
+ private final Map<String, Object> superConf;
+ private final IStormClusterState stormClusterState;
+ private final EventManager syncSupEventManager;
+ private final AtomicReference<Map<String, VersionedData<Assignment>>> assignmentVersions;
+ private final Map<Integer, Slot> slots = new HashMap<>();
+ private final AtomicInteger readRetry = new AtomicInteger(0);
+ private final String assignmentId;
+ private final ISupervisor iSuper;
+ private final ILocalizer localizer;
+ private final ContainerLauncher launcher;
+ private final String host;
+ private final LocalState localState;
+ private final IStormClusterState clusterState;
+ private final AtomicReference<Map<Long, LocalAssignment>> cachedAssignments;
+
+ public ReadClusterState(Supervisor supervisor) throws Exception {
+ this.superConf = supervisor.getConf();
+ this.stormClusterState = supervisor.getStormClusterState();
+ this.syncSupEventManager = supervisor.getEventManger();
+ this.assignmentVersions = new AtomicReference<>(new HashMap<>());
+ this.assignmentId = supervisor.getAssignmentId();
+ this.iSuper = supervisor.getiSupervisor();
+ this.localizer = supervisor.getAsyncLocalizer();
+ this.host = supervisor.getHostName();
+ this.localState = supervisor.getLocalState();
+ this.clusterState = supervisor.getStormClusterState();
+ this.cachedAssignments = supervisor.getCurrAssignment();
+
+ this.launcher = ContainerLauncher.make(superConf, assignmentId, supervisor.getSharedContext());
+
+ @SuppressWarnings("unchecked")
+ List<Number> ports = (List<Number>)superConf.get(Config.SUPERVISOR_SLOTS_PORTS);
+ for (Number port: ports) {
+ slots.put(port.intValue(), mkSlot(port.intValue()));
+ }
+
+ try {
+ Collection<String> workers = SupervisorUtils.supervisorWorkerIds(superConf);
+ for (Slot slot: slots.values()) {
+ String workerId = slot.getWorkerId();
+ if (workerId != null) {
+ workers.remove(workerId);
+ }
+ }
+ if (!workers.isEmpty()) {
+ supervisor.killWorkers(workers, launcher);
+ }
+ } catch (Exception e) {
+ LOG.warn("Error trying to clean up old workers", e);
+ }
+
+ //All the slots/assignments should be recovered now, so we can clean up anything that we don't expect to be here
+ try {
+ localizer.cleanupUnusedTopologies();
+ } catch (Exception e) {
+ LOG.warn("Error trying to clean up old topologies", e);
+ }
+
+ for (Slot slot: slots.values()) {
+ slot.start();
+ }
+ }
+
+ private Slot mkSlot(int port) throws Exception {
+ return new Slot(localizer, superConf, launcher, host, port,
+ localState, clusterState, iSuper, cachedAssignments);
+ }
+
+ @Override
+ public synchronized void run() {
+ try {
+ Runnable syncCallback = new EventManagerPushCallback(this, syncSupEventManager);
+ List<String> stormIds = stormClusterState.assignments(syncCallback);
+ Map<String, VersionedData<Assignment>> assignmentsSnapshot =
+ getAssignmentsSnapshot(stormClusterState, stormIds, assignmentVersions.get(), syncCallback);
+
+ Map<Integer, LocalAssignment> allAssignments =
+ readAssignments(assignmentsSnapshot);
+ if (allAssignments == null) {
+ //Something odd happened try again later
+ return;
+ }
+ Map<String, List<ProfileRequest>> topoIdToProfilerActions = getProfileActions(stormClusterState, stormIds);
+
+ HashSet<Integer> assignedPorts = new HashSet<>();
+ LOG.debug("Synchronizing supervisor");
+ LOG.debug("All assignment: {}", allAssignments);
+ LOG.debug("Topology Ids -> Profiler Actions {}", topoIdToProfilerActions);
+ for (Integer port: allAssignments.keySet()) {
+ if (iSuper.confirmAssigned(port)) {
+ assignedPorts.add(port);
+ }
+ }
+ HashSet<Integer> allPorts = new HashSet<>(assignedPorts);
+ allPorts.addAll(slots.keySet());
+
+ Map<Integer, Set<TopoProfileAction>> filtered = new HashMap<>();
+ for (Entry<String, List<ProfileRequest>> entry: topoIdToProfilerActions.entrySet()) {
+ String topoId = entry.getKey();
+ if (entry.getValue() != null) {
+ for (ProfileRequest req: entry.getValue()) {
+ NodeInfo ni = req.get_nodeInfo();
+ if (host.equals(ni.get_node())) {
+ Long port = ni.get_port().iterator().next();
+ Set<TopoProfileAction> actions = filtered.get(port);
+ if (actions == null) {
+ actions = new HashSet<>();
+ filtered.put(port.intValue(), actions);
+ }
+ actions.add(new TopoProfileAction(topoId, req));
+ }
+ }
+ }
+ }
+
+ for (Integer port: allPorts) {
+ Slot slot = slots.get(port);
+ if (slot == null) {
+ slot = mkSlot(port);
+ slots.put(port, slot);
+ slot.start();
+ }
+ slot.setNewAssignment(allAssignments.get(port));
+ slot.addProfilerActions(filtered.get(port));
+ }
+
+ } catch (Exception e) {
+ LOG.error("Failed to Sync Supervisor", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected Map<String, VersionedData<Assignment>> getAssignmentsSnapshot(IStormClusterState stormClusterState, List<String> topoIds,
+ Map<String, VersionedData<Assignment>> localAssignmentVersion, Runnable callback) throws Exception {
+ Map<String, VersionedData<Assignment>> updateAssignmentVersion = new HashMap<>();
+ for (String topoId : topoIds) {
+ Integer recordedVersion = -1;
+ Integer version = stormClusterState.assignmentVersion(topoId, callback);
+ VersionedData<Assignment> locAssignment = localAssignmentVersion.get(topoId);
+ if (locAssignment != null) {
+ recordedVersion = locAssignment.getVersion();
+ }
+ if (version == null) {
+ // ignore
+ } else if (version == recordedVersion) {
+ updateAssignmentVersion.put(topoId, locAssignment);
+ } else {
+ VersionedData<Assignment> assignmentVersion = stormClusterState.assignmentInfoWithVersion(topoId, callback);
+ updateAssignmentVersion.put(topoId, assignmentVersion);
+ }
+ }
+ return updateAssignmentVersion;
+ }
+
+ protected Map<String, List<ProfileRequest>> getProfileActions(IStormClusterState stormClusterState, List<String> stormIds) throws Exception {
+ Map<String, List<ProfileRequest>> ret = new HashMap<String, List<ProfileRequest>>();
+ for (String stormId : stormIds) {
+ List<ProfileRequest> profileRequests = stormClusterState.getTopologyProfileRequests(stormId);
+ ret.put(stormId, profileRequests);
+ }
+ return ret;
+ }
+
+ protected Map<Integer, LocalAssignment> readAssignments(Map<String, VersionedData<Assignment>> assignmentsSnapshot) {
+ try {
+ Map<Integer, LocalAssignment> portLA = new HashMap<>();
+ for (Map.Entry<String, VersionedData<Assignment>> assignEntry : assignmentsSnapshot.entrySet()) {
+ String topoId = assignEntry.getKey();
+ Assignment assignment = assignEntry.getValue().getData();
+
+ Map<Integer, LocalAssignment> portTasks = readMyExecutors(topoId, assignmentId, assignment);
+
+ for (Map.Entry<Integer, LocalAssignment> entry : portTasks.entrySet()) {
+
+ Integer port = entry.getKey();
+
+ LocalAssignment la = entry.getValue();
+
+ if (!portLA.containsKey(port)) {
+ portLA.put(port, la);
+ } else {
+ throw new RuntimeException("Should not have multiple topologies assigned to one port "
+ + port + " " + la + " " + portLA);
+ }
+ }
+ }
+ readRetry.set(0);
+ return portLA;
+ } catch (RuntimeException e) {
+ if (readRetry.get() > 2) {
+ throw e;
+ } else {
+ readRetry.addAndGet(1);
+ }
+ LOG.warn("{} : retrying {} of 3", e.getMessage(), readRetry.get());
+ return null;
+ }
+ }
+
+ protected Map<Integer, LocalAssignment> readMyExecutors(String stormId, String assignmentId, Assignment assignment) {
+ Map<Integer, LocalAssignment> portTasks = new HashMap<>();
+ Map<Long, WorkerResources> slotsResources = new HashMap<>();
+ Map<NodeInfo, WorkerResources> nodeInfoWorkerResourcesMap = assignment.get_worker_resources();
+ if (nodeInfoWorkerResourcesMap != null) {
+ for (Map.Entry<NodeInfo, WorkerResources> entry : nodeInfoWorkerResourcesMap.entrySet()) {
+ if (entry.getKey().get_node().equals(assignmentId)) {
+ Set<Long> ports = entry.getKey().get_port();
+ for (Long port : ports) {
+ slotsResources.put(port, entry.getValue());
+ }
+ }
+ }
+ }
+ Map<List<Long>, NodeInfo> executorNodePort = assignment.get_executor_node_port();
+ if (executorNodePort != null) {
+ for (Map.Entry<List<Long>, NodeInfo> entry : executorNodePort.entrySet()) {
+ if (entry.getValue().get_node().equals(assignmentId)) {
+ for (Long port : entry.getValue().get_port()) {
+ LocalAssignment localAssignment = portTasks.get(port.intValue());
+ if (localAssignment == null) {
+ List<ExecutorInfo> executors = new ArrayList<>();
+ localAssignment = new LocalAssignment(stormId, executors);
+ if (slotsResources.containsKey(port)) {
+ localAssignment.set_resources(slotsResources.get(port));
+ }
+ portTasks.put(port.intValue(), localAssignment);
+ }
+ List<ExecutorInfo> executorInfoList = localAssignment.get_executors();
+ executorInfoList.add(new ExecutorInfo(entry.getKey().get(0).intValue(), entry.getKey().get(entry.getKey().size() - 1).intValue()));
+ }
+ }
+ }
+ }
+ return portTasks;
+ }
+
+ public synchronized void shutdownAllWorkers() {
+ for (Slot slot: slots.values()) {
+ slot.setNewAssignment(null);
+ }
+
+ for (Slot slot: slots.values()) {
+ try {
+ int count = 0;
+ while (slot.getMachineState() != MachineState.EMPTY) {
+ if (count > 10) {
+ LOG.warn("DONE waiting for {} to finish {}", slot, slot.getMachineState());
+ break;
+ }
+ if (Time.isSimulating()) {
+ Time.advanceTime(1000);
+ Thread.sleep(100);
+ } else {
+ Time.sleep(100);
+ }
+ count++;
+ }
+ } catch (Exception e) {
+ LOG.error("Error trying to shutdown workers in {}", slot, e);
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ for (Slot slot: slots.values()) {
+ try {
+ slot.close();
+ } catch (Exception e) {
+ LOG.error("Error trying to shutdown {}", slot, e);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainer.java
new file mode 100644
index 0000000..29b8576
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainer.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RunAsUserContainer extends BasicContainer {
+ private static final Logger LOG = LoggerFactory.getLogger(RunAsUserContainer.class);
+
+ public RunAsUserContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+ LocalAssignment assignment, LocalState localState,
+ String workerId) throws IOException {
+ this(type, conf, supervisorId, port, assignment, localState, workerId, null, null, null);
+ }
+
+ RunAsUserContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+ LocalAssignment assignment, LocalState localState,
+ String workerId, Map<String, Object> topoConf, AdvancedFSOps ops, String profileCmd) throws IOException {
+ super(type, conf, supervisorId, port, assignment, localState, workerId, topoConf, ops,
+ profileCmd);
+ if (Utils.isOnWindows()) {
+ throw new UnsupportedOperationException("ERROR: Windows doesn't support running workers as different users yet");
+ }
+ }
+
+ private void signal(long pid, int signal) throws IOException {
+ List<String> commands = Arrays.asList("signal", String.valueOf(pid), String.valueOf(signal));
+ String user = getWorkerUser();
+ String logPrefix = "kill -"+signal+" " + pid;
+ SupervisorUtils.processLauncherAndWait(_conf, user, commands, null, logPrefix);
+ }
+
+ @Override
+ protected void kill(long pid) throws IOException {
+ signal(pid, 15);
+ }
+
+ @Override
+ protected void forceKill(long pid) throws IOException {
+ signal(pid, 9);
+ }
+
+ @Override
+ protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix, File targetDir) throws IOException, InterruptedException {
+ String user = this.getWorkerUser();
+ String td = targetDir.getAbsolutePath();
+ LOG.info("Running as user: {} command: {}", user, command);
+ String containerFile = Utils.containerFilePath(td);
+ if (Utils.checkFileExists(containerFile)) {
+ SupervisorUtils.rmrAsUser(_conf, containerFile, containerFile);
+ }
+ String scriptFile = Utils.scriptFilePath(td);
+ if (Utils.checkFileExists(scriptFile)) {
+ SupervisorUtils.rmrAsUser(_conf, scriptFile, scriptFile);
+ }
+ String script = Utils.writeScript(td, command, env);
+ List<String> args = Arrays.asList("profiler", td, script);
+ int ret = SupervisorUtils.processLauncherAndWait(_conf, user, args, env, logPrefix);
+ return ret == 0;
+ }
+
+ @Override
+ protected void launchWorkerProcess(List<String> command, Map<String, String> env,
+ String logPrefix, ExitCodeCallback processExitCallback, File targetDir) throws IOException {
+ String workerDir = targetDir.getAbsolutePath();
+ String user = this.getWorkerUser();
+ List<String> args = Arrays.asList("worker", workerDir, Utils.writeScript(workerDir, command, env));
+ List<String> commandPrefix = null;
+ SupervisorUtils.processLauncher(_conf, user, commandPrefix, args, null, logPrefix, processExitCallback, targetDir);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java
new file mode 100644
index 0000000..5b446b3
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.storm.daemon.supervisor.Container.ContainerType;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.utils.LocalState;
+
+public class RunAsUserContainerLauncher extends ContainerLauncher {
+ private final Map<String, Object> _conf;
+ private final String _supervisorId;
+
+ public RunAsUserContainerLauncher(Map<String, Object> conf, String supervisorId) throws IOException {
+ _conf = conf;
+ _supervisorId = supervisorId;
+ }
+
+ @Override
+ public Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+ Container container = new RunAsUserContainer(ContainerType.LAUNCH, _conf, _supervisorId, port, assignment,
+ state, null, null, null, null);
+ container.setup();
+ container.launch();
+ return container;
+ }
+
+ @Override
+ public Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+ return new RunAsUserContainer(ContainerType.RECOVER_FULL, _conf, _supervisorId, port, assignment,
+ state, null, null, null, null);
+ }
+
+ @Override
+ public Killable recoverContainer(String workerId, LocalState localState) throws IOException {
+ return new RunAsUserContainer(ContainerType.RECOVER_PARTIAL, _conf, _supervisorId, -1, null,
+ localState, workerId, null, null, null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Slot.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Slot.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Slot.java
new file mode 100644
index 0000000..c92db0c
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Slot.java
@@ -0,0 +1,785 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.storm.Config;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.localizer.ILocalizer;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Slot extends Thread implements AutoCloseable {
+ private static final Logger LOG = LoggerFactory.getLogger(Slot.class);
+
+ static enum MachineState {
+ EMPTY,
+ RUNNING,
+ WAITING_FOR_WORKER_START,
+ KILL_AND_RELAUNCH,
+ KILL,
+ WAITING_FOR_BASIC_LOCALIZATION,
+ WAITING_FOR_BLOB_LOCALIZATION;
+ };
+
+ static class StaticState {
+ public final ILocalizer localizer;
+ public final long hbTimeoutMs;
+ public final long firstHbTimeoutMs;
+ public final long killSleepMs;
+ public final long monitorFreqMs;
+ public final ContainerLauncher containerLauncher;
+ public final int port;
+ public final String host;
+ public final ISupervisor iSupervisor;
+ public final LocalState localState;
+
+ StaticState(ILocalizer localizer, long hbTimeoutMs, long firstHbTimeoutMs,
+ long killSleepMs, long monitorFreqMs,
+ ContainerLauncher containerLauncher, String host, int port,
+ ISupervisor iSupervisor, LocalState localState) {
+ this.localizer = localizer;
+ this.hbTimeoutMs = hbTimeoutMs;
+ this.firstHbTimeoutMs = firstHbTimeoutMs;
+ this.containerLauncher = containerLauncher;
+ this.killSleepMs = killSleepMs;
+ this.monitorFreqMs = monitorFreqMs;
+ this.host = host;
+ this.port = port;
+ this.iSupervisor = iSupervisor;
+ this.localState = localState;
+ }
+ }
+
+ static class DynamicState {
+ public final MachineState state;
+ public final LocalAssignment newAssignment;
+ public final LocalAssignment currentAssignment;
+ public final Container container;
+ public final LocalAssignment pendingLocalization;
+ public final Future<Void> pendingDownload;
+ public final Set<TopoProfileAction> profileActions;
+ public final Set<TopoProfileAction> pendingStopProfileActions;
+
+ /**
+ * The last time that WAITING_FOR_WORKER_START, KILL, or KILL_AND_RELAUNCH were entered into.
+ */
+ public final long startTime;
+
+ public DynamicState(final LocalAssignment currentAssignment, Container container, final LocalAssignment newAssignment) {
+ this.currentAssignment = currentAssignment;
+ this.container = container;
+ if ((currentAssignment == null) ^ (container == null)) {
+ throw new IllegalArgumentException("Container and current assignment must both be null, or neither can be null");
+ }
+
+ if (currentAssignment == null) {
+ state = MachineState.EMPTY;
+ } else {
+ state = MachineState.RUNNING;
+ }
+
+ this.startTime = System.currentTimeMillis();
+ this.newAssignment = newAssignment;
+ this.pendingLocalization = null;
+ this.pendingDownload = null;
+ this.profileActions = new HashSet<>();
+ this.pendingStopProfileActions = new HashSet<>();
+ }
+
+ public DynamicState(final MachineState state, final LocalAssignment newAssignment,
+ final Container container, final LocalAssignment currentAssignment,
+ final LocalAssignment pendingLocalization, final long startTime,
+ final Future<Void> pendingDownload, final Set<TopoProfileAction> profileActions,
+ final Set<TopoProfileAction> pendingStopProfileActions) {
+ this.state = state;
+ this.newAssignment = newAssignment;
+ this.currentAssignment = currentAssignment;
+ this.container = container;
+ this.pendingLocalization = pendingLocalization;
+ this.startTime = startTime;
+ this.pendingDownload = pendingDownload;
+ this.profileActions = profileActions;
+ this.pendingStopProfileActions = pendingStopProfileActions;
+ }
+
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(state);
+ sb.append(" msInState: ");
+ sb.append(Time.currentTimeMillis() - startTime);
+ if (container != null) {
+ sb.append(" ");
+ sb.append(container);
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Set the new assignment for the state. This should never be called from within the state machine.
+ * It is an input from outside.
+ * @param newAssignment the new assignment to set
+ * @return the updated DynamicState.
+ */
+ public DynamicState withNewAssignment(LocalAssignment newAssignment) {
+ return new DynamicState(this.state, newAssignment,
+ this.container, this.currentAssignment,
+ this.pendingLocalization, this.startTime,
+ this.pendingDownload, this.profileActions,
+ this.pendingStopProfileActions);
+ }
+
+ public DynamicState withPendingLocalization(LocalAssignment pendingLocalization, Future<Void> pendingDownload) {
+ return new DynamicState(this.state, this.newAssignment,
+ this.container, this.currentAssignment,
+ pendingLocalization, this.startTime,
+ pendingDownload, this.profileActions,
+ this.pendingStopProfileActions);
+ }
+
+ public DynamicState withPendingLocalization(Future<Void> pendingDownload) {
+ return withPendingLocalization(this.pendingLocalization, pendingDownload);
+ }
+
+ public DynamicState withState(final MachineState state) {
+ long newStartTime = Time.currentTimeMillis();
+ return new DynamicState(state, this.newAssignment,
+ this.container, this.currentAssignment,
+ this.pendingLocalization, newStartTime,
+ this.pendingDownload, this.profileActions,
+ this.pendingStopProfileActions);
+ }
+
+ public DynamicState withCurrentAssignment(final Container container, final LocalAssignment currentAssignment) {
+ return new DynamicState(this.state, this.newAssignment,
+ container, currentAssignment,
+ this.pendingLocalization, this.startTime,
+ this.pendingDownload, this.profileActions,
+ this.pendingStopProfileActions);
+ }
+
+ public DynamicState withProfileActions(Set<TopoProfileAction> profileActions, Set<TopoProfileAction> pendingStopProfileActions) {
+ return new DynamicState(this.state, this.newAssignment,
+ this.container, this.currentAssignment,
+ this.pendingLocalization, this.startTime,
+ this.pendingDownload, profileActions,
+ pendingStopProfileActions);
+ }
+ };
+
+ static class TopoProfileAction {
+ public final String topoId;
+ public final ProfileRequest request;
+
+ public TopoProfileAction(String topoId, ProfileRequest request) {
+ this.topoId = topoId;
+ this.request = request;
+ }
+
+ @Override
+ public int hashCode() {
+ return (37 * topoId.hashCode()) + request.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof TopoProfileAction)) {
+ return false;
+ }
+ TopoProfileAction o = (TopoProfileAction) other;
+ return topoId.equals(o.topoId) && request.equals(o.request);
+ }
+
+ @Override
+ public String toString() {
+ return "{ " + topoId + ": " + request + " }";
+ }
+ }
+
+ static boolean equivalent(LocalAssignment a, LocalAssignment b) {
+ if (a == null && b == null) {
+ return true;
+ }
+ if (a != null && b != null) {
+ if (a.get_topology_id().equals(b.get_topology_id())) {
+ Set<ExecutorInfo> aexec = new HashSet<>(a.get_executors());
+ Set<ExecutorInfo> bexec = new HashSet<>(b.get_executors());
+ if (aexec.equals(bexec)) {
+ boolean aHasResources = a.is_set_resources();
+ boolean bHasResources = b.is_set_resources();
+ if (!aHasResources && !bHasResources) {
+ return true;
+ }
+ if (aHasResources && bHasResources) {
+ if (a.get_resources().equals(b.get_resources())) {
+ return true;
+ }
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ static DynamicState stateMachineStep(DynamicState dynamicState, StaticState staticState) throws Exception {
+ LOG.debug("STATE {}", dynamicState.state);
+ switch (dynamicState.state) {
+ case EMPTY:
+ return handleEmpty(dynamicState, staticState);
+ case RUNNING:
+ return handleRunning(dynamicState, staticState);
+ case WAITING_FOR_WORKER_START:
+ return handleWaitingForWorkerStart(dynamicState, staticState);
+ case KILL_AND_RELAUNCH:
+ return handleKillAndRelaunch(dynamicState, staticState);
+ case KILL:
+ return handleKill(dynamicState, staticState);
+ case WAITING_FOR_BASIC_LOCALIZATION:
+ return handleWaitingForBasicLocalization(dynamicState, staticState);
+ case WAITING_FOR_BLOB_LOCALIZATION:
+ return handleWaitingForBlobLocalization(dynamicState, staticState);
+ default:
+ throw new IllegalStateException("Code not ready to handle a state of "+dynamicState.state);
+ }
+ }
+
+ /**
+ * Prepare for a new assignment by downloading new required blobs, or going to empty if there is nothing to download.
+ * PRECONDITION: The slot should be empty
+ * @param dynamicState current state
+ * @param staticState static data
+ * @return the next state
+ * @throws IOException on any error
+ */
+ static DynamicState prepareForNewAssignmentNoWorkersRunning(DynamicState dynamicState, StaticState staticState) throws IOException {
+ assert(dynamicState.container == null);
+
+ if (dynamicState.newAssignment == null) {
+ return dynamicState.withState(MachineState.EMPTY);
+ }
+ Future<Void> pendingDownload = staticState.localizer.requestDownloadBaseTopologyBlobs(dynamicState.newAssignment, staticState.port);
+ return dynamicState.withPendingLocalization(dynamicState.newAssignment, pendingDownload).withState(MachineState.WAITING_FOR_BASIC_LOCALIZATION);
+ }
+
+ /**
+ * Kill the current container and start downloading what the new assignment needs, if there is a new assignment
+ * PRECONDITION: container != null
+ * @param dynamicState current state
+ * @param staticState static data
+ * @return the next state
+ * @throws Exception
+ */
+ static DynamicState killContainerForChangedAssignment(DynamicState dynamicState, StaticState staticState) throws Exception {
+ assert(dynamicState.container != null);
+
+ staticState.iSupervisor.killedWorker(staticState.port);
+ dynamicState.container.kill();
+ Future<Void> pendingDownload = null;
+ if (dynamicState.newAssignment != null) {
+ pendingDownload = staticState.localizer.requestDownloadBaseTopologyBlobs(dynamicState.newAssignment, staticState.port);
+ }
+ Time.sleep(staticState.killSleepMs);
+ return dynamicState.withPendingLocalization(dynamicState.newAssignment, pendingDownload).withState(MachineState.KILL);
+ }
+
+ /**
+ * Kill the current container and relaunch it. (Something odd happened)
+ * PRECONDITION: container != null
+ * @param dynamicState current state
+ * @param staticState static data
+ * @return the next state
+ * @throws Exception
+ */
+ static DynamicState killAndRelaunchContainer(DynamicState dynamicState, StaticState staticState) throws Exception {
+ assert(dynamicState.container != null);
+
+ dynamicState.container.kill();
+ Time.sleep(staticState.killSleepMs);
+
+ //any stop profile actions that hadn't timed out yet, we should restart after the worker is running again.
+ HashSet<TopoProfileAction> mod = new HashSet<>(dynamicState.profileActions);
+ mod.addAll(dynamicState.pendingStopProfileActions);
+ return dynamicState.withState(MachineState.KILL_AND_RELAUNCH).withProfileActions(mod, Collections.<TopoProfileAction> emptySet());
+ }
+
+ /**
+ * Clean up a container
+ * PRECONDITION: All of the processes have died.
+ * @param dynamicState current state
+ * @param staticState static data
+ * @param nextState the next MachineState to go to.
+ * @return the next state.
+ */
+ static DynamicState cleanupCurrentContainer(DynamicState dynamicState, StaticState staticState, MachineState nextState) throws Exception {
+ assert(dynamicState.container != null);
+ assert(dynamicState.currentAssignment != null);
+ assert(dynamicState.container.areAllProcessesDead());
+
+ dynamicState.container.cleanUp();
+ staticState.localizer.releaseSlotFor(dynamicState.currentAssignment, staticState.port);
+ DynamicState ret = dynamicState.withCurrentAssignment(null, null);
+ if (nextState != null) {
+ ret = ret.withState(nextState);
+ }
+ return ret;
+ }
+
+ /**
+ * State Transitions for WAITING_FOR_BLOB_LOCALIZATION state.
+ * PRECONDITION: neither pendingLocalization nor pendingDownload is null.
+ * PRECONDITION: The slot should be empty
+ * @param dynamicState current state
+ * @param staticState static data
+ * @return the next state
+ * @throws Exception on any error
+ */
+ static DynamicState handleWaitingForBlobLocalization(DynamicState dynamicState, StaticState staticState) throws Exception {
+ assert(dynamicState.pendingLocalization != null);
+ assert(dynamicState.pendingDownload != null);
+ assert(dynamicState.container == null);
+
+ //Ignore changes to scheduling while downloading the topology blobs
+ // We don't support canceling the download through the future yet,
+ // so to keep everything in sync, just wait
+ try {
+ dynamicState.pendingDownload.get(1000, TimeUnit.MILLISECONDS);
+ //Downloading of all blobs finished.
+ if (!equivalent(dynamicState.newAssignment, dynamicState.pendingLocalization)) {
+ //Scheduling changed
+ staticState.localizer.releaseSlotFor(dynamicState.pendingLocalization, staticState.port);
+ return prepareForNewAssignmentNoWorkersRunning(dynamicState, staticState);
+ }
+ Container c = staticState.containerLauncher.launchContainer(staticState.port, dynamicState.pendingLocalization, staticState.localState);
+ return dynamicState.withCurrentAssignment(c, dynamicState.pendingLocalization).withState(MachineState.WAITING_FOR_WORKER_START).withPendingLocalization(null, null);
+ } catch (TimeoutException e) {
+ //We waited for 1 second loop around and try again....
+ return dynamicState;
+ }
+ }
+
+ /**
+ * State Transitions for WAITING_FOR_BASIC_LOCALIZATION state.
+ * PRECONDITION: neither pendingLocalization nor pendingDownload is null.
+ * PRECONDITION: The slot should be empty
+ * @param dynamicState current state
+ * @param staticState static data
+ * @return the next state
+ * @throws Exception on any error
+ */
+ static DynamicState handleWaitingForBasicLocalization(DynamicState dynamicState, StaticState staticState) throws Exception {
+ assert(dynamicState.pendingLocalization != null);
+ assert(dynamicState.pendingDownload != null);
+ assert(dynamicState.container == null);
+
+ //Ignore changes to scheduling while downloading the topology code
+ // We don't support canceling the download through the future yet,
+ // so to keep everything in sync, just wait
+ try {
+ dynamicState.pendingDownload.get(1000, TimeUnit.MILLISECONDS);
+ Future<Void> pendingDownload = staticState.localizer.requestDownloadTopologyBlobs(dynamicState.pendingLocalization, staticState.port);
+ return dynamicState.withPendingLocalization(pendingDownload).withState(MachineState.WAITING_FOR_BLOB_LOCALIZATION);
+ } catch (TimeoutException e) {
+ return dynamicState;
+ }
+ }
+
+ /**
+ * State Transitions for KILL state.
+ * PRECONDITION: container.kill() was called
+ * PRECONDITION: container != null && currentAssignment != null
+ * @param dynamicState current state
+ * @param staticState static data
+ * @return the next state
+ * @throws Exception on any error
+ */
+ static DynamicState handleKill(DynamicState dynamicState, StaticState staticState) throws Exception {
+ assert(dynamicState.container != null);
+ assert(dynamicState.currentAssignment != null);
+
+ if (dynamicState.container.areAllProcessesDead()) {
+ LOG.warn("SLOT {} all processes are dead...", staticState.port);
+ return cleanupCurrentContainer(dynamicState, staticState,
+ dynamicState.pendingLocalization == null ? MachineState.EMPTY : MachineState.WAITING_FOR_BASIC_LOCALIZATION);
+ }
+
+ LOG.warn("SLOT {} force kill and wait...", staticState.port);
+ dynamicState.container.forceKill();
+ Time.sleep(staticState.killSleepMs);
+ return dynamicState;
+ }
+
+ /**
+ * State Transitions for KILL_AND_RELAUNCH state.
+ * PRECONDITION: container.kill() was called
+ * PRECONDITION: container != null && currentAssignment != null
+ * @param dynamicState current state
+ * @param staticState static data
+ * @return the next state
+ * @throws Exception on any error
+ */
+ static DynamicState handleKillAndRelaunch(DynamicState dynamicState, StaticState staticState) throws Exception {
+ assert(dynamicState.container != null);
+ assert(dynamicState.currentAssignment != null);
+
+ if (dynamicState.container.areAllProcessesDead()) {
+ if (equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
+ dynamicState.container.cleanUpForRestart();
+ dynamicState.container.relaunch();
+ return dynamicState.withState(MachineState.WAITING_FOR_WORKER_START);
+ }
+ //Scheduling changed after we killed all of the processes
+ return prepareForNewAssignmentNoWorkersRunning(cleanupCurrentContainer(dynamicState, staticState, null), staticState);
+ }
+ //The child processes typically exit in < 1 sec. If 2 mins later they are still around something is wrong
+ if ((Time.currentTimeMillis() - dynamicState.startTime) > 120_000) {
+ throw new RuntimeException("Not all processes in " + dynamicState.container + " exited after 120 seconds");
+ }
+ dynamicState.container.forceKill();
+ Time.sleep(staticState.killSleepMs);
+ return dynamicState;
+ }
+
+ /**
+ * State Transitions for WAITING_FOR_WORKER_START state.
+ * PRECONDITION: container != null && currentAssignment != null
+ * @param dynamicState current state
+ * @param staticState static data
+ * @return the next state
+ * @throws Exception on any error
+ */
+ static DynamicState handleWaitingForWorkerStart(DynamicState dynamicState, StaticState staticState) throws Exception {
+ assert(dynamicState.container != null);
+ assert(dynamicState.currentAssignment != null);
+
+ LSWorkerHeartbeat hb = dynamicState.container.readHeartbeat();
+ if (hb != null) {
+ long hbAgeMs = (Time.currentTimeSecs() - hb.get_time_secs()) * 1000;
+ if (hbAgeMs <= staticState.hbTimeoutMs) {
+ return dynamicState.withState(MachineState.RUNNING);
+ }
+ }
+
+ if (!equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
+ //We were rescheduled while waiting for the worker to come up
+ LOG.warn("SLOT {}: Assignment Changed from {} to {}", staticState.port, dynamicState.currentAssignment, dynamicState.newAssignment);
+ return killContainerForChangedAssignment(dynamicState, staticState);
+ }
+
+ long timeDiffms = (Time.currentTimeMillis() - dynamicState.startTime);
+ if (timeDiffms > staticState.firstHbTimeoutMs) {
+ LOG.warn("SLOT {}: Container {} failed to launch in {} ms.", staticState.port, dynamicState.container, staticState.firstHbTimeoutMs);
+ return killAndRelaunchContainer(dynamicState, staticState);
+ }
+ Time.sleep(1000);
+ return dynamicState;
+ }
+
+ /**
+ * State Transitions for RUNNING state.
+ * PRECONDITION: container != null && currentAssignment != null
+ * @param dynamicState current state
+ * @param staticState static data
+ * @return the next state
+ * @throws Exception on any error
+ */
+ static DynamicState handleRunning(DynamicState dynamicState, StaticState staticState) throws Exception {
+ assert(dynamicState.container != null);
+ assert(dynamicState.currentAssignment != null);
+
+ if (!equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
+ LOG.warn("SLOT {}: Assignment Changed from {} to {}", staticState.port, dynamicState.currentAssignment, dynamicState.newAssignment);
+ //Scheduling changed while running...
+ return killContainerForChangedAssignment(dynamicState, staticState);
+ }
+ if (dynamicState.container.didMainProcessExit()) {
+ LOG.warn("SLOT {}: main process has exited", staticState.port);
+ return killAndRelaunchContainer(dynamicState, staticState);
+ }
+
+ LSWorkerHeartbeat hb = dynamicState.container.readHeartbeat();
+ if (hb == null) {
+ LOG.warn("SLOT {}: HB returned as null", staticState.port);
+ //This can happen if the supervisor crashed after launching a
+ // worker that never came up.
+ return killAndRelaunchContainer(dynamicState, staticState);
+ }
+
+ long timeDiffMs = (Time.currentTimeSecs() - hb.get_time_secs()) * 1000;
+ if (timeDiffMs > staticState.hbTimeoutMs) {
+ LOG.warn("SLOT {}: HB is too old {} > {}", staticState.port, timeDiffMs, staticState.hbTimeoutMs);
+ return killAndRelaunchContainer(dynamicState, staticState);
+ }
+
+ //The worker is up and running check for profiling requests
+ if (!dynamicState.profileActions.isEmpty()) {
+ HashSet<TopoProfileAction> mod = new HashSet<>(dynamicState.profileActions);
+ HashSet<TopoProfileAction> modPending = new HashSet<>(dynamicState.pendingStopProfileActions);
+ Iterator<TopoProfileAction> iter = mod.iterator();
+ while (iter.hasNext()) {
+ TopoProfileAction action = iter.next();
+ if (!action.topoId.equals(dynamicState.currentAssignment.get_topology_id())) {
+ iter.remove();
+ LOG.warn("Dropping {} wrong topology is running", action);
+ //Not for this topology so skip it
+ } else {
+ if (modPending.contains(action)) {
+ boolean isTimeForStop = Time.currentTimeMillis() > action.request.get_time_stamp();
+ if (isTimeForStop) {
+ if (dynamicState.container.runProfiling(action.request, true)) {
+ LOG.debug("Stopped {} action finished", action);
+ iter.remove();
+ modPending.remove(action);
+ } else {
+ LOG.warn("Stopping {} failed, will be retried", action);
+ }
+ } else {
+ LOG.debug("Still pending {} now: {}", action, Time.currentTimeMillis());
+ }
+ } else {
+ //J_PROFILE_START is not used. When you see a J_PROFILE_STOP
+ // start profiling and save it away to stop when timeout happens
+ if (action.request.get_action() == ProfileAction.JPROFILE_STOP) {
+ if (dynamicState.container.runProfiling(action.request, false)) {
+ modPending.add(action);
+ LOG.debug("Started {} now: {}", action, Time.currentTimeMillis());
+ } else {
+ LOG.warn("Starting {} failed, will be retried", action);
+ }
+ } else {
+ if (dynamicState.container.runProfiling(action.request, false)) {
+ LOG.debug("Started {} action finished", action);
+ iter.remove();
+ } else {
+ LOG.warn("Starting {} failed, will be retried", action);
+ }
+ }
+ }
+ }
+ }
+ dynamicState = dynamicState.withProfileActions(mod, modPending);
+ }
+ Time.sleep(staticState.monitorFreqMs);
+ return dynamicState;
+ }
+
+ static DynamicState handleEmpty(DynamicState dynamicState, StaticState staticState) throws InterruptedException, IOException {
+ if (!equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
+ return prepareForNewAssignmentNoWorkersRunning(dynamicState, staticState);
+ }
+ //Both assignments are null, just wait
+ if (dynamicState.profileActions != null && !dynamicState.profileActions.isEmpty()) {
+ //Nothing is scheduled here so throw away all of the profileActions
+ LOG.warn("Dropping {} no topology is running", dynamicState.profileActions);
+ dynamicState = dynamicState.withProfileActions(Collections.<TopoProfileAction> emptySet(), Collections.<TopoProfileAction> emptySet());
+ }
+ Time.sleep(1000);
+ return dynamicState;
+ }
+
+ private final AtomicReference<LocalAssignment> newAssignment = new AtomicReference<>();
+ private final AtomicReference<Set<TopoProfileAction>> profiling =
+ new AtomicReference<Set<TopoProfileAction>>(new HashSet<TopoProfileAction>());
+ private final StaticState staticState;
+ private final IStormClusterState clusterState;
+ private volatile boolean done = false;
+ private volatile DynamicState dynamicState;
+ private final AtomicReference<Map<Long, LocalAssignment>> cachedCurrentAssignments;
+
+ public Slot(ILocalizer localizer, Map<String, Object> conf,
+ ContainerLauncher containerLauncher, String host,
+ int port, LocalState localState,
+ IStormClusterState clusterState,
+ ISupervisor iSupervisor,
+ AtomicReference<Map<Long, LocalAssignment>> cachedCurrentAssignments) throws Exception {
+ super("SLOT_"+port);
+
+ this.cachedCurrentAssignments = cachedCurrentAssignments;
+ this.clusterState = clusterState;
+ Map<Integer, LocalAssignment> assignments = localState.getLocalAssignmentsMap();
+ LocalAssignment currentAssignment = null;
+ if (assignments != null) {
+ currentAssignment = assignments.get(port);
+ }
+ Container container = null;
+ if (currentAssignment != null) {
+ try {
+ container = containerLauncher.recoverContainer(port, currentAssignment, localState);
+ } catch (ContainerRecoveryException e) {
+ //We could not recover container will be null.
+ }
+ }
+
+ LocalAssignment newAssignment = currentAssignment;
+ if (currentAssignment != null && container == null) {
+ currentAssignment = null;
+ //Assigned something but it is not running
+ }
+
+ dynamicState = new DynamicState(currentAssignment, container, newAssignment);
+ staticState = new StaticState(localizer,
+ Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)) * 1000,
+ Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_START_TIMEOUT_SECS)) * 1000,
+ Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS)) * 1000,
+ Utils.getInt(conf.get(Config.SUPERVISOR_MONITOR_FREQUENCY_SECS)) * 1000,
+ containerLauncher,
+ host,
+ port,
+ iSupervisor,
+ localState);
+ this.newAssignment.set(dynamicState.newAssignment);
+ if (MachineState.RUNNING == dynamicState.state) {
+ //We are running so we should recover the blobs.
+ staticState.localizer.recoverRunningTopology(currentAssignment, port);
+ saveNewAssignment(currentAssignment);
+ }
+ LOG.warn("SLOT {}:{} Starting in state {} - assignment {}", staticState.host, staticState.port, dynamicState.state, dynamicState.currentAssignment);
+ }
+
+ public MachineState getMachineState() {
+ return dynamicState.state;
+ }
+
+ /**
+ * Set a new assignment asynchronously
+ * @param newAssignment the new assignment for this slot to run, null to run nothing
+ */
+ public void setNewAssignment(LocalAssignment newAssignment) {
+ this.newAssignment.set(newAssignment);
+ }
+
+ public void addProfilerActions(Set<TopoProfileAction> actions) {
+ if (actions != null) {
+ while(true) {
+ Set<TopoProfileAction> orig = profiling.get();
+ Set<TopoProfileAction> newActions = new HashSet<>(orig);
+ newActions.addAll(actions);
+ if (profiling.compareAndSet(orig, newActions)) {
+ return;
+ }
+ }
+ }
+ }
+
+ public String getWorkerId() {
+ String workerId = null;
+ Container c = dynamicState.container;
+ if (c != null) {
+ workerId = c.getWorkerId();
+ }
+ return workerId;
+ }
+
+ private void saveNewAssignment(LocalAssignment assignment) {
+ synchronized(staticState.localState) {
+ Map<Integer, LocalAssignment> assignments = staticState.localState.getLocalAssignmentsMap();
+ if (assignments == null) {
+ assignments = new HashMap<>();
+ }
+ if (assignment == null) {
+ assignments.remove(staticState.port);
+ } else {
+ assignments.put(staticState.port, assignment);
+ }
+ staticState.localState.setLocalAssignmentsMap(assignments);
+ }
+ Map<Long, LocalAssignment> update = null;
+ Map<Long, LocalAssignment> orig = null;
+ do {
+ Long lport = new Long(staticState.port);
+ orig = cachedCurrentAssignments.get();
+ update = new HashMap<>(orig);
+ if (assignment == null) {
+ update.remove(lport);
+ } else {
+ update.put(lport, assignment);
+ }
+ } while (!cachedCurrentAssignments.compareAndSet(orig, update));
+ }
+
+ public void run() {
+ try {
+ while(!done) {
+ Set<TopoProfileAction> origProfileActions = new HashSet<>(profiling.get());
+ Set<TopoProfileAction> removed = new HashSet<>(origProfileActions);
+
+ DynamicState nextState =
+ stateMachineStep(dynamicState.withNewAssignment(newAssignment.get())
+ .withProfileActions(origProfileActions, dynamicState.pendingStopProfileActions), staticState);
+
+ if (LOG.isDebugEnabled() || dynamicState.state != nextState.state) {
+ LOG.info("STATE {} -> {}", dynamicState, nextState);
+ }
+ //Save the current state for recovery
+ if (!equivalent(nextState.currentAssignment, dynamicState.currentAssignment)) {
+ LOG.info("SLOT {}: Changing current assignment from {} to {}", staticState.port, dynamicState.currentAssignment, nextState.currentAssignment);
+ saveNewAssignment(nextState.currentAssignment);
+ }
+
+ // clean up the profiler actions that are not being processed
+ removed.removeAll(dynamicState.profileActions);
+ removed.removeAll(dynamicState.pendingStopProfileActions);
+ for (TopoProfileAction action: removed) {
+ try {
+ clusterState.deleteTopologyProfileRequests(action.topoId, action.request);
+ } catch (Exception e) {
+ LOG.error("Error trying to remove profiling request, it will be retried", e);
+ }
+ }
+ Set<TopoProfileAction> orig, copy;
+ do {
+ orig = profiling.get();
+ copy = new HashSet<>(orig);
+ copy.removeAll(removed);
+ } while (!profiling.compareAndSet(orig, copy));
+ dynamicState = nextState;
+ }
+ } catch (Throwable e) {
+ if (!Utils.exceptionCauseIsInstanceOf(InterruptedException.class, e)) {
+ LOG.error("Error when processing event", e);
+ Utils.exitProcess(20, "Error when processing an event");
+ }
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ done = true;
+ this.interrupt();
+ this.join();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java
new file mode 100644
index 0000000..c14b060
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+public class StandaloneSupervisor implements ISupervisor {
+ private String supervisorId;
+ private Map conf;
+
+ @Override
+ public void prepare(Map stormConf, String schedulerLocalDir) {
+ try {
+ LocalState localState = new LocalState(schedulerLocalDir);
+ String supervisorId = localState.getSupervisorId();
+ if (supervisorId == null) {
+ supervisorId = generateSupervisorId();
+ localState.setSupervisorId(supervisorId);
+ }
+ this.conf = stormConf;
+ this.supervisorId = supervisorId;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public String getSupervisorId() {
+ return supervisorId;
+ }
+
+ @Override
+ public String getAssignmentId() {
+ return supervisorId;
+ }
+
+ @Override
+ public Object getMetadata() {
+ Object ports = conf.get(Config.SUPERVISOR_SLOTS_PORTS);
+ return ports;
+ }
+
+ @Override
+ public boolean confirmAssigned(int port) {
+ return true;
+ }
+
+ @Override
+ public void killedWorker(int port) {
+
+ }
+
+ @Override
+ public void assigned(Collection<Integer> ports) {
+
+ }
+
+ public String generateSupervisorId(){
+ return Utils.uuid();
+ }
+}
[19/21] storm git commit: Removed healthcheck.clj. Is not needed
Posted by ka...@apache.org.
Removed healthcheck.clj. Is not needed
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/df33efd6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/df33efd6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/df33efd6
Branch: refs/heads/1.x-branch
Commit: df33efd698318d70c4c798cc31748828992aae50
Parents: 81cd074
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Oct 26 15:43:37 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Oct 26 15:43:37 2016 -0500
----------------------------------------------------------------------
.../org/apache/storm/command/healthcheck.clj | 88 --------------------
1 file changed, 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/df33efd6/storm-core/src/clj/org/apache/storm/command/healthcheck.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/command/healthcheck.clj b/storm-core/src/clj/org/apache/storm/command/healthcheck.clj
deleted file mode 100644
index d96d7b3..0000000
--- a/storm-core/src/clj/org/apache/storm/command/healthcheck.clj
+++ /dev/null
@@ -1,88 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.command.healthcheck
- (:require [org.apache.storm
- [config :refer :all]
- [log :refer :all]]
- [clojure.java [io :as io]]
- [clojure [string :refer [split]]])
- (:gen-class))
-
-(defn interrupter
- "Interrupt a given thread after ms milliseconds."
- [thread ms]
- (let [interrupter (Thread.
- (fn []
- (try
- (Thread/sleep ms)
- (.interrupt thread)
- (catch InterruptedException e))))]
- (.start interrupter)
- interrupter))
-
-(defn check-output [lines]
- (if (some #(.startsWith % "ERROR") lines)
- :failed
- :success))
-
-(defn process-script [conf script]
- (let [script-proc (. (Runtime/getRuntime) (exec script))
- curthread (Thread/currentThread)
- interrupter-thread (interrupter curthread
- (conf STORM-HEALTH-CHECK-TIMEOUT-MS))]
- (try
- (.waitFor script-proc)
- (.interrupt interrupter-thread)
- (if (not (= (.exitValue script-proc) 0))
- :failed_with_exit_code
- (check-output (split
- (slurp (.getInputStream script-proc))
- #"\n+")))
- (catch InterruptedException e
- (println "Script" script "timed out.")
- :timeout)
- (catch Exception e
- (println "Script failed with exception: " e)
- :failed_with_exception)
- (finally (.interrupt interrupter-thread)))))
-
-(defn health-check [conf]
- (let [health-dir (absolute-healthcheck-dir conf)
- health-files (file-seq (io/file health-dir))
- health-scripts (filter #(and (.canExecute %)
- (not (.isDirectory %)))
- health-files)
- results (->> health-scripts
- (map #(.getAbsolutePath %))
- (map (partial process-script conf)))]
- (log-message
- (pr-str (map #'vector
- (map #(.getAbsolutePath %) health-scripts)
- results)))
- ; failed_with_exit_code is OK. We're mimicing Hadoop's health checks.
- ; We treat non-zero exit codes as indicators that the scripts failed
- ; to execute properly, not that the system is unhealthy, in which case
- ; we don't want to start killing things.
- (if (every? #(or (= % :failed_with_exit_code)
- (= % :success))
- results)
- 0
- 1)))
-
-(defn -main [& args]
- (let [conf (read-storm-config)]
- (System/exit
- (health-check conf))))
[18/21] storm git commit: Merge branch 'STORM-2131' of
https://github.com/knusbaum/incubator-storm
Posted by ka...@apache.org.
Merge branch 'STORM-2131' of https://github.com/knusbaum/incubator-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/81cd0745
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/81cd0745
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/81cd0745
Branch: refs/heads/1.x-branch
Commit: 81cd0745260ca066d9cf5e73ac4a34fa9ef60d55
Parents: 1d4d414
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Tue Oct 11 11:01:31 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Oct 25 14:10:35 2016 -0500
----------------------------------------------------------------------
.../storm/daemon/supervisor/AdvancedFSOps.java | 15 +++++++
.../storm/daemon/supervisor/Container.java | 2 +-
.../daemon/supervisor/SupervisorUtils.java | 12 ++++-
.../src/native/worker-launcher/impl/main.c | 20 ++++++++-
.../worker-launcher/impl/worker-launcher.c | 46 ++++++++++++--------
.../worker-launcher/impl/worker-launcher.h | 2 +-
6 files changed, 75 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
index 361328e..e531b84 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
@@ -95,6 +95,11 @@ public class AdvancedFSOps {
public void setupStormCodeDir(Map<String, Object> topologyConf, File path) throws IOException {
SupervisorUtils.setupStormCodeDir(_conf, topologyConf, path.getCanonicalPath());
}
+
+ @Override
+ public void setupWorkerArtifactsDir(Map<String, Object> topologyConf, File path) throws IOException {
+ SupervisorUtils.setupWorkerArtifactsDir(_conf, topologyConf, path.getCanonicalPath());
+ }
}
/**
@@ -225,6 +230,16 @@ public class AdvancedFSOps {
}
/**
+ * Setup the permissions for the worker artifacts dirs
+ * @param topologyConf the config of the Topology
+ * @param path the directory to set the permissions on
+ * @throws IOException on any error
+ */
+ public void setupWorkerArtifactsDir(Map<String, Object> topologyConf, File path) throws IOException {
+ //By default this is a NOOP
+ }
+
+ /**
* Sanity check if everything the topology needs is there for it to run.
* @param conf the config of the supervisor
* @param topologyId the ID of the topology
http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
index aea4b53..f4eb5e0 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
@@ -312,7 +312,7 @@ public abstract class Container implements Killable {
File workerArtifacts = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port));
if (!_ops.fileExists(workerArtifacts)) {
_ops.forceMkdir(workerArtifacts);
- _ops.setupStormCodeDir(_topoConf, workerArtifacts);
+ _ops.setupWorkerArtifactsDir(_topoConf, workerArtifacts);
}
String user = getWorkerUser();
http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
index 0784631..19d3b78 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
@@ -96,7 +96,7 @@ public class SupervisorUtils {
public static void setupStormCodeDir(Map<String, Object> conf, Map<String, Object> stormConf, String dir) throws IOException {
if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
- String logPrefix = "setup conf for " + dir;
+ String logPrefix = "Storm Code Dir Setup for " + dir;
List<String> commands = new ArrayList<>();
commands.add("code-dir");
commands.add(dir);
@@ -104,6 +104,16 @@ public class SupervisorUtils {
}
}
+ public static void setupWorkerArtifactsDir(Map<String, Object> conf, Map<String, Object> stormConf, String dir) throws IOException {
+ if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
+ String logPrefix = "Worker Artifacts Setup for " + dir;
+ List<String> commands = new ArrayList<>();
+ commands.add("artifacts-dir");
+ commands.add(dir);
+ processLauncherAndWait(conf, (String) (stormConf.get(Config.TOPOLOGY_SUBMITTER_USER)), commands, null, logPrefix);
+ }
+ }
+
public static void rmrAsUser(Map<String, Object> conf, String id, String path) throws IOException {
String user = Utils.getFileOwner(path);
String logPreFix = "rmr " + id;
http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/native/worker-launcher/impl/main.c
----------------------------------------------------------------------
diff --git a/storm-core/src/native/worker-launcher/impl/main.c b/storm-core/src/native/worker-launcher/impl/main.c
index a51f9f9..b14db45 100644
--- a/storm-core/src/native/worker-launcher/impl/main.c
+++ b/storm-core/src/native/worker-launcher/impl/main.c
@@ -155,7 +155,23 @@ int main(int argc, char **argv) {
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
}
- exit_code = setup_stormdist_dir(argv[optind]);
+ exit_code = setup_dir_permissions(argv[optind], 0);
+ } else if (strcasecmp("artifacts-dir", command) == 0) {
+ if (argc != 4) {
+ fprintf(ERRORFILE, "Incorrect number of arguments (%d vs 4) for artifacts-dir\n",
+ argc);
+ fflush(ERRORFILE);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ exit_code = setup_dir_permissions(argv[optind], 1);
+ } else if (strcasecmp("blob", command) == 0) {
+ if (argc != 4) {
+ fprintf(ERRORFILE, "Incorrect number of arguments (%d vs 4) for blob\n",
+ argc);
+ fflush(ERRORFILE);
+ return INVALID_ARGUMENT_NUMBER;
+ }
+ exit_code = setup_dir_permissions(argv[optind], 0);
} else if (strcasecmp("rmr", command) == 0) {
if (argc != 4) {
fprintf(ERRORFILE, "Incorrect number of arguments (%d vs 4) for rmr\n",
@@ -173,7 +189,7 @@ int main(int argc, char **argv) {
return INVALID_ARGUMENT_NUMBER;
}
working_dir = argv[optind++];
- exit_code = setup_stormdist_dir(working_dir);
+ exit_code = setup_dir_permissions(working_dir, 1);
if (exit_code == 0) {
exit_code = exec_as_user(working_dir, argv[optind]);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/native/worker-launcher/impl/worker-launcher.c
----------------------------------------------------------------------
diff --git a/storm-core/src/native/worker-launcher/impl/worker-launcher.c b/storm-core/src/native/worker-launcher/impl/worker-launcher.c
index 5dc90d3..7baec01 100644
--- a/storm-core/src/native/worker-launcher/impl/worker-launcher.c
+++ b/storm-core/src/native/worker-launcher/impl/worker-launcher.c
@@ -414,33 +414,45 @@ static int copy_file(int input, const char* in_filename,
return 0;
}
-int setup_stormdist(FTSENT* entry, uid_t euser) {
+/**
+ * Sets up permissions for a directory optionally making it user-writable.
+ * We set up the permissions r(w)xrws--- so that the file group (should be Storm's user group)
+ * has complete access to the directory, and the file user (The topology owner's user)
+ * is able to read and execute, and in certain directories, write. The setGID bit is set
+ * to make sure any files created under the directory will be accessible to storm's user for
+ * cleanup purposes.
+ */
+static int setup_permissions(FTSENT* entry, uid_t euser, int user_write) {
if (lchown(entry->fts_path, euser, launcher_gid) != 0) {
- fprintf(ERRORFILE, "Failure to exec app initialization process - %s\n",
- strerror(errno));
+ fprintf(ERRORFILE, "Failure to exec app initialization process - %s, fts_path=%s\n",
+ strerror(errno), entry->fts_path);
return -1;
}
mode_t mode = entry->fts_statp->st_mode;
- mode_t new_mode = (mode & (S_IRWXU)) | S_IRGRP | S_IWGRP;
- if ((mode & S_IXUSR) == S_IXUSR) {
- new_mode = new_mode | S_IXGRP;
+ // Preserve user read and execute and set group read and write.
+ mode_t new_mode = (mode & (S_IRUSR | S_IXUSR)) | S_IRGRP | S_IWGRP;
+ if (user_write) {
+ new_mode = new_mode | S_IWUSR;
}
+ // If the entry is a directory, Add group execute and setGID bits.
if ((mode & S_IFDIR) == S_IFDIR) {
- new_mode = new_mode | S_ISGID;
+ new_mode = new_mode | S_IXGRP | S_ISGID;
}
if (chmod(entry->fts_path, new_mode) != 0) {
- fprintf(ERRORFILE, "Failure to exec app initialization process - %s\n",
- strerror(errno));
+ fprintf(ERRORFILE, "Failure to exec app initialization process - %s, fts_path=%s\n",
+ strerror(errno), entry->fts_path);
return -1;
}
return 0;
}
-int setup_stormdist_dir(const char* local_dir) {
+
+int setup_dir_permissions(const char* local_dir, int user_writable) {
//This is the same as
//> chmod g+rwX -R $local_dir
- //> chown -no-dereference -R $user:$supervisor-group $local_dir
-
+ //> chmod g+s -R $local_dir
+ //> if [ $user_writable ]; then chmod u+w; else u-w; fi
+ //> chown -no-dereference -R $user:$supervisor-group $local_dir
int exit_code = 0;
uid_t euser = geteuid();
@@ -450,7 +462,7 @@ int setup_stormdist_dir(const char* local_dir) {
} else {
char *(paths[]) = {strndup(local_dir,PATH_MAX), 0};
if (paths[0] == NULL) {
- fprintf(ERRORFILE, "Malloc failed in setup_stormdist_dir\n");
+ fprintf(ERRORFILE, "Malloc failed in setup_dir_permissions\n");
return -1;
}
// check to make sure the directory exists
@@ -485,14 +497,14 @@ int setup_stormdist_dir(const char* local_dir) {
case FTS_DP: // A directory being visited in post-order
case FTS_DOT: // A dot directory
+ case FTS_SL: // A symbolic link
+ case FTS_SLNONE: // A broken symbolic link
//NOOP
fprintf(LOGFILE, "NOOP: %s\n", entry->fts_path); break;
case FTS_D: // A directory in pre-order
case FTS_F: // A regular file
- case FTS_SL: // A symbolic link
- case FTS_SLNONE: // A broken symbolic link
- if (setup_stormdist(entry, euser) != 0) {
- exit_code = -1;
+ if (setup_permissions(entry, euser, user_writable) != 0) {
+ exit_code = -1;
}
break;
case FTS_DEFAULT: // Unknown type of file
http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/native/worker-launcher/impl/worker-launcher.h
----------------------------------------------------------------------
diff --git a/storm-core/src/native/worker-launcher/impl/worker-launcher.h b/storm-core/src/native/worker-launcher/impl/worker-launcher.h
index 3b1ec24..4a0b657 100644
--- a/storm-core/src/native/worker-launcher/impl/worker-launcher.h
+++ b/storm-core/src/native/worker-launcher/impl/worker-launcher.h
@@ -66,7 +66,7 @@ extern FILE *LOGFILE;
// the log file for error messages
extern FILE *ERRORFILE;
-int setup_stormdist_dir(const char* local_dir);
+int setup_dir_permissions(const char* local_dir, int for_blob_permission);
int exec_as_user(const char * working_dir, const char * args);
[10/21] storm git commit: STORM-2018: Move back to JDK 1.7
Posted by ka...@apache.org.
STORM-2018: Move back to JDK 1.7
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/24ff529b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/24ff529b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/24ff529b
Branch: refs/heads/1.x-branch
Commit: 24ff529b8b3fcb99956a897aacfd96f4b85bba7d
Parents: 318ab5f
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Sep 21 10:04:50 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Sep 21 10:04:50 2016 -0500
----------------------------------------------------------------------
pom.xml | 4 +--
.../storm/daemon/supervisor/BasicContainer.java | 15 +++++---
.../daemon/supervisor/ReadClusterState.java | 2 +-
.../storm/daemon/supervisor/Supervisor.java | 7 +++-
.../daemon/supervisor/BasicContainerTest.java | 37 +++++++++++---------
.../storm/daemon/supervisor/ContainerTest.java | 2 +-
.../storm/localizer/AsyncLocalizerTest.java | 9 +++--
7 files changed, 48 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0665c10..a1466c9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1067,8 +1067,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
- <source>1.8</source>
- <target>1.8</target>
+ <source>1.7</source>
+ <target>1.7</target>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
index 237d89b..c5b40f2 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -52,7 +52,12 @@ import com.google.common.collect.Lists;
*/
public class BasicContainer extends Container {
private static final Logger LOG = LoggerFactory.getLogger(BasicContainer.class);
- private static final FilenameFilter jarFilter = (dir, name) -> name.endsWith(".jar");
+ private static final FilenameFilter jarFilter = new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.endsWith(".jar");
+ }
+ };
private static final Joiner CPJ =
Joiner.on(Utils.CLASS_PATH_SEPARATOR).skipNulls();
@@ -334,9 +339,11 @@ public class BasicContainer extends Container {
if (files == null) {
return Collections.emptyList();
}
-
- return Arrays.stream(files).map(f -> f.getAbsolutePath())
- .collect(Collectors.toList());
+ ArrayList<String> ret = new ArrayList<>(files.length);
+ for (File f: files) {
+ ret.add(f.getAbsolutePath());
+ }
+ return ret;
}
protected List<String> frameworkClasspath() {
http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
index 27f18b6..40b4a93 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
@@ -69,7 +69,7 @@ public class ReadClusterState implements Runnable, AutoCloseable {
this.superConf = supervisor.getConf();
this.stormClusterState = supervisor.getStormClusterState();
this.syncSupEventManager = supervisor.getEventManger();
- this.assignmentVersions = new AtomicReference<>(new HashMap<>());
+ this.assignmentVersions = new AtomicReference<Map<String, VersionedData<Assignment>>>(new HashMap<String, VersionedData<Assignment>>());
this.assignmentId = supervisor.getAssignmentId();
this.iSuper = supervisor.getiSupervisor();
this.localizer = supervisor.getAsyncLocalizer();
http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
index 5e89eea..1399e8d 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
@@ -241,7 +241,12 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
throw new IllegalArgumentException("Cannot start server in local mode!");
}
launch();
- Utils.addShutdownHookWithForceKillIn1Sec(() -> {this.close();});
+ Utils.addShutdownHookWithForceKillIn1Sec(new Runnable(){
+ @Override
+ public void run() {
+ close();
+ }
+ });
registerWorkerNumGauge("supervisor:num-slots-used-gauge", conf);
StormMetricsRegistry.startMetricsReporters(conf);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
index edf534a..38081f4 100644
--- a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
@@ -31,9 +31,12 @@ import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.daemon.supervisor.Container.ContainerType;
+import org.apache.storm.generated.Bolt;
import org.apache.storm.generated.LocalAssignment;
import org.apache.storm.generated.ProfileAction;
import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.generated.SpoutSpec;
+import org.apache.storm.generated.StateSpoutSpec;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.utils.LocalState;
import org.apache.storm.utils.Utils;
@@ -126,7 +129,7 @@ public class BasicContainerTest {
LocalState ls = mock(LocalState.class);
MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", port, la, ls, null, new HashMap<>(), ops, "profile");
+ "SUPERVISOR", port, la, ls, null, new HashMap<String, Object>(), ops, "profile");
//null worker id means generate one...
assertNotNull(mc._workerId);
@@ -155,7 +158,7 @@ public class BasicContainerTest {
when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
MockBasicContainer mc = new MockBasicContainer(ContainerType.RECOVER_FULL, superConf,
- "SUPERVISOR", port, la, ls, null, new HashMap<>(), ops, "profile");
+ "SUPERVISOR", port, la, ls, null, new HashMap<String, Object>(), ops, "profile");
assertEquals(workerId, mc._workerId);
}
@@ -175,7 +178,7 @@ public class BasicContainerTest {
try {
new MockBasicContainer(ContainerType.RECOVER_FULL, new HashMap<String, Object>(),
- "SUPERVISOR", port, la, ls, null, new HashMap<>(), null, "profile");
+ "SUPERVISOR", port, la, ls, null, new HashMap<String, Object>(), null, "profile");
fail("Container recovered worker incorrectly");
} catch (ContainerRecoveryException e) {
//Expected
@@ -201,7 +204,7 @@ public class BasicContainerTest {
when(ls.getApprovedWorkers()).thenReturn(new HashMap<>(workerState));
MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+ "SUPERVISOR", port, la, ls, workerId, new HashMap<String, Object>(), ops, "profile");
mc.cleanUp();
@@ -235,7 +238,7 @@ public class BasicContainerTest {
LocalState ls = mock(LocalState.class);
MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+ "SUPERVISOR", port, la, ls, workerId, new HashMap<String, Object>(), ops, "profile");
//HEAP DUMP
ProfileRequest req = new ProfileRequest();
@@ -369,9 +372,9 @@ public class BasicContainerTest {
final String workerTmpDir = ContainerTest.asAbsPath(workerRoot, "tmp");
final StormTopology st = new StormTopology();
- st.set_spouts(new HashMap<>());
- st.set_bolts(new HashMap<>());
- st.set_state_spouts(new HashMap<>());
+ st.set_spouts(new HashMap<String, SpoutSpec>());
+ st.set_bolts(new HashMap<String, Bolt>());
+ st.set_state_spouts(new HashMap<String, StateSpoutSpec>());
byte [] serializedState = Utils.gzip(Utils.thriftSerialize(st));
final Map<String, Object> superConf = new HashMap<>();
@@ -379,19 +382,21 @@ public class BasicContainerTest {
superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
superConf.put(Config.STORM_LOG4J2_CONF_DIR, log4jdir);
- LocalAssignment la = new LocalAssignment();
+ final LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
- AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ final AdvancedFSOps ops = mock(AdvancedFSOps.class);
when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
when(ops.slurp(stormcode)).thenReturn(serializedState);
- LocalState ls = mock(LocalState.class);
+ final LocalState ls = mock(LocalState.class);
- checkpoint(() -> {
+ checkpoint(new Run() {
+ @Override
+ public void run() throws Exception {
MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+ "SUPERVISOR", port, la, ls, workerId, new HashMap<String, Object>(), ops, "profile");
mc.launch();
@@ -440,7 +445,7 @@ public class BasicContainerTest {
workerId
), cmd.cmd);
assertEquals(new File(workerRoot), cmd.pwd);
- },
+ }},
"storm.home", stormHome,
"storm.log.dir", stormLogDir);
}
@@ -463,7 +468,7 @@ public class BasicContainerTest {
LocalState ls = mock(LocalState.class);
MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+ "SUPERVISOR", port, la, ls, workerId, new HashMap<String, Object>(), ops, "profile");
assertListEquals(Arrays.asList(
"-Xloggc:/tmp/storm/logs/gc.worker-9999-s-01-w-01-9999.log",
@@ -477,7 +482,7 @@ public class BasicContainerTest {
"-Xmx512m"),
mc.substituteChildopts(Arrays.asList("-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log","-Xms256m","-Xmx%HEAP-MEM%m"), memOnheap));
- assertListEquals(Collections.emptyList(),
+ assertListEquals(Collections.<String>emptyList(),
mc.substituteChildopts(null));
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
index 9b20ef8..3f34552 100644
--- a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
@@ -101,7 +101,7 @@ public class ContainerTest {
LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf,
- "SUPERVISOR", 8080, la, "worker", new HashMap<>(), ops);
+ "SUPERVISOR", 8080, la, "worker", new HashMap<String, Object>(), ops);
mc.kill();
assertEquals(Collections.EMPTY_LIST, mc.killedPids);
assertEquals(Collections.EMPTY_LIST, mc.forceKilledPids);
http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java b/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
index 41c17c8..ed2b4ff 100644
--- a/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
@@ -34,8 +34,11 @@ import org.junit.Test;
import org.apache.storm.Config;
import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.generated.Bolt;
import org.apache.storm.generated.ExecutorInfo;
import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.SpoutSpec;
+import org.apache.storm.generated.StateSpoutSpec;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.localizer.LocalizedResource;
import org.apache.storm.localizer.Localizer;
@@ -128,9 +131,9 @@ public class AsyncLocalizerTest {
final String simpleCurrentLocalFile = localizerRoot + user + "/simple.current";
final StormTopology st = new StormTopology();
- st.set_spouts(new HashMap<>());
- st.set_bolts(new HashMap<>());
- st.set_state_spouts(new HashMap<>());
+ st.set_spouts(new HashMap<String, SpoutSpec>());
+ st.set_bolts(new HashMap<String, Bolt>());
+ st.set_state_spouts(new HashMap<String, StateSpoutSpec>());
Map<String, Map<String, Object>> topoBlobMap = new HashMap<>();
Map<String, Object> simple = new HashMap<>();
[03/21] storm git commit: STORM-2018: Supervisor V2
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/Utils.java b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
index 7a322ec..9323db4 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/Utils.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
@@ -17,26 +17,16 @@
*/
package org.apache.storm.utils;
-import org.apache.storm.Config;
-import org.apache.storm.blobstore.BlobStore;
-import org.apache.storm.blobstore.BlobStoreAclHandler;
-import org.apache.storm.blobstore.ClientBlobStore;
-import org.apache.storm.blobstore.InputStreamWithMeta;
-import org.apache.storm.blobstore.LocalFsBlobStore;
-import org.apache.storm.daemon.JarTransformer;
-import org.apache.storm.generated.*;
-import org.apache.storm.localizer.Localizer;
-import org.apache.storm.nimbus.NimbusInfo;
-import org.apache.storm.serialization.DefaultSerializationDelegate;
-import org.apache.storm.serialization.SerializationDelegate;
-
-import clojure.lang.IFn;
-import clojure.lang.RT;
-
+import clojure.lang.Keyword;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.exec.CommandLine;
+import org.apache.commons.exec.DefaultExecutor;
+import org.apache.commons.exec.ExecuteException;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.ClassLoaderObjectInputStream;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.ensemble.exhibitor.DefaultExhibitorRestClient;
@@ -44,6 +34,31 @@ import org.apache.curator.ensemble.exhibitor.ExhibitorEnsembleProvider;
import org.apache.curator.ensemble.exhibitor.Exhibitors;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.storm.Config;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.blobstore.BlobStoreAclHandler;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.blobstore.InputStreamWithMeta;
+import org.apache.storm.blobstore.LocalFsBlobStore;
+import org.apache.storm.daemon.JarTransformer;
+import org.apache.storm.generated.AccessControl;
+import org.apache.storm.generated.AccessControlType;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.ClusterSummary;
+import org.apache.storm.generated.ComponentCommon;
+import org.apache.storm.generated.ComponentObject;
+import org.apache.storm.generated.GlobalStreamId;
+import org.apache.storm.generated.KeyNotFoundException;
+import org.apache.storm.generated.Nimbus;
+import org.apache.storm.generated.ReadableBlobMeta;
+import org.apache.storm.generated.SettableBlobMeta;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.generated.TopologyInfo;
+import org.apache.storm.generated.TopologySummary;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.nimbus.NimbusInfo;
+import org.apache.storm.serialization.DefaultSerializationDelegate;
+import org.apache.storm.serialization.SerializationDelegate;
import org.apache.thrift.TBase;
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
@@ -61,12 +76,15 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
+import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -77,13 +95,16 @@ import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.RandomAccessFile;
import java.io.Serializable;
+import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.URL;
import java.net.URLDecoder;
import java.net.UnknownHostException;
+import java.net.ServerSocket;
import java.nio.ByteBuffer;
import java.nio.file.FileSystems;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermission;
import java.util.ArrayList;
@@ -94,11 +115,14 @@ import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
+import java.util.concurrent.Callable;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.regex.Matcher;
@@ -108,8 +132,27 @@ import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
+import clojure.lang.RT;
+
public class Utils {
- private static final Logger LOG = LoggerFactory.getLogger(Utils.class);
+ // A singleton instance allows us to mock delegated static methods in our
+ // tests by subclassing.
+ private static Utils _instance = new Utils();
+
+ /**
+ * Provide an instance of this class for delegates to use. To mock out
+ * delegated methods, provide an instance of a subclass that overrides the
+ * implementation of the delegated method.
+ * @param u a Utils instance
+ * @return the previously set instance
+ */
+ public static Utils setInstance(Utils u) {
+ Utils oldInstance = _instance;
+ _instance = u;
+ return oldInstance;
+ }
+
+ public static final Logger LOG = LoggerFactory.getLogger(Utils.class);
public static final String DEFAULT_STREAM_ID = "default";
public static final String DEFAULT_BLOB_VERSION_SUFFIX = ".version";
public static final String CURRENT_BLOB_SUFFIX_ID = "current";
@@ -120,15 +163,35 @@ public class Utils {
private static SerializationDelegate serializationDelegate;
private static ClassLoader cl = null;
+ public static final boolean IS_ON_WINDOWS = "Windows_NT".equals(System.getenv("OS"));
+ public static final String FILE_PATH_SEPARATOR = System.getProperty("file.separator");
+ public static final String CLASS_PATH_SEPARATOR = System.getProperty("path.separator");
+
+ public static final int SIGKILL = 9;
+ public static final int SIGTERM = 15;
+
static {
- Map conf = readStormConfig();
+ Map<String, Object> conf = readStormConfig();
serializationDelegate = getSerializationDelegate(conf);
}
- public static Object newInstance(String klass) {
+ @SuppressWarnings("unchecked")
+ public static <T> T newInstance(String klass) {
+ try {
+ return newInstance((Class<T>)Class.forName(klass));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static <T> T newInstance(Class<T> klass) {
+ return _instance.newInstanceImpl(klass);
+ }
+
+ // Non-static impl methods exist for mocking purposes.
+ public <T> T newInstanceImpl(Class<T> klass) {
try {
- Class c = Class.forName(klass);
- return c.newInstance();
+ return klass.newInstance();
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -150,11 +213,11 @@ public class Utils {
return serializationDelegate.deserialize(serialized, clazz);
}
- public static <T> T thriftDeserialize(Class c, byte[] b, int offset, int length) {
+ public static <T> T thriftDeserialize(Class<T> c, byte[] b, int offset, int length) {
try {
- T ret = (T) c.newInstance();
+ T ret = c.newInstance();
TDeserializer des = getDes();
- des.deserialize((TBase)ret, b, offset, length);
+ des.deserialize((TBase) ret, b, offset, length);
return ret;
} catch (Exception e) {
throw new RuntimeException(e);
@@ -261,7 +324,6 @@ public class Utils {
return ret.toString();
}
-
public static long bitXorVals(List<Long> coll) {
long result = 0;
for (Long val : coll) {
@@ -291,16 +353,17 @@ public class Utils {
}
}
- public static Map findAndReadConfigFile(String name, boolean mustExist) {
+ public static Map<String, Object> findAndReadConfigFile(String name, boolean mustExist) {
InputStream in = null;
boolean confFileEmpty = false;
try {
in = getConfigFileInputStream(name);
if (null != in) {
Yaml yaml = new Yaml(new SafeConstructor());
- Map ret = (Map) yaml.load(new InputStreamReader(in));
+ @SuppressWarnings("unchecked")
+ Map<String, Object> ret = (Map<String, Object>) yaml.load(new InputStreamReader(in));
if (null != ret) {
- return new HashMap(ret);
+ return new HashMap<>(ret);
} else {
confFileEmpty = true;
}
@@ -312,7 +375,7 @@ public class Utils {
else
throw new RuntimeException("Could not find config file on classpath " + name);
} else {
- return new HashMap();
+ return new HashMap<>();
}
} catch (IOException e) {
throw new RuntimeException(e);
@@ -354,16 +417,16 @@ public class Utils {
}
- public static Map findAndReadConfigFile(String name) {
+ public static Map<String, Object> findAndReadConfigFile(String name) {
return findAndReadConfigFile(name, true);
}
- public static Map readDefaultConfig() {
+ public static Map<String, Object> readDefaultConfig() {
return findAndReadConfigFile("defaults.yaml", true);
}
- public static Map readCommandLineOpts() {
- Map ret = new HashMap();
+ public static Map<String, Object> readCommandLineOpts() {
+ Map<String, Object> ret = new HashMap<>();
String commandOptions = System.getProperty("storm.options");
if (commandOptions != null) {
/*
@@ -394,10 +457,10 @@ public class Utils {
return ret;
}
- public static Map readStormConfig() {
- Map ret = readDefaultConfig();
+ public static Map<String, Object> readStormConfig() {
+ Map<String, Object> ret = readDefaultConfig();
String confFile = System.getProperty("storm.conf.file");
- Map storm;
+ Map<String, Object> storm;
if (confFile == null || confFile.equals("")) {
storm = findAndReadConfigFile("storm.yaml", false);
} else {
@@ -487,7 +550,11 @@ public class Utils {
HashMap nconf = new HashMap(conf);
// only enable cleanup of blobstore on nimbus
nconf.put(Config.BLOBSTORE_CLEANUP_ENABLE, Boolean.TRUE);
- store.prepare(nconf, baseDir, nimbusInfo);
+
+ if(store != null) {
+ // store can be null during testing when mocking utils.
+ store.prepare(nconf, baseDir, nimbusInfo);
+ }
return store;
}
@@ -502,6 +569,11 @@ public class Utils {
*/
public static void downloadResourcesAsSupervisor(String key, String localFile,
ClientBlobStore cb) throws AuthorizationException, KeyNotFoundException, IOException {
+ _instance.downloadResourcesAsSupervisorImpl(key, localFile, cb);
+ }
+
+ public void downloadResourcesAsSupervisorImpl(String key, String localFile,
+ ClientBlobStore cb) throws AuthorizationException, KeyNotFoundException, IOException {
final int MAX_RETRY_ATTEMPTS = 2;
final int ATTEMPTS_INTERVAL_TIME = 100;
for (int retryAttempts = 0; retryAttempts < MAX_RETRY_ATTEMPTS; retryAttempts++) {
@@ -520,11 +592,8 @@ public class Utils {
private static boolean downloadResourcesAsSupervisorAttempt(ClientBlobStore cb, String key, String localFile) {
boolean isSuccess = false;
- FileOutputStream out = null;
- InputStreamWithMeta in = null;
- try {
- out = new FileOutputStream(localFile);
- in = cb.getBlob(key);
+ try (FileOutputStream out = new FileOutputStream(localFile);
+ InputStreamWithMeta in = cb.getBlob(key);) {
long fileSize = in.getFileLength();
byte[] buffer = new byte[1024];
@@ -538,17 +607,6 @@ public class Utils {
isSuccess = (fileSize == downloadFileSize);
} catch (TException | IOException e) {
LOG.error("An exception happened while downloading {} from blob store.", localFile, e);
- } finally {
- try {
- if (out != null) {
- out.close();
- }
- } catch (IOException ignored) {}
- try {
- if (in != null) {
- in.close();
- }
- } catch (IOException ignored) {}
}
if (!isSuccess) {
try {
@@ -560,8 +618,21 @@ public class Utils {
return isSuccess;
}
+ public static boolean checkFileExists(File path) {
+ return Files.exists(path.toPath());
+ }
+
+ public static boolean checkFileExists(String path) {
+ return Files.exists(new File(path).toPath());
+ }
+
public static boolean checkFileExists(String dir, String file) {
- return Files.exists(new File(dir, file).toPath());
+ return checkFileExists(dir + FILE_PATH_SEPARATOR + file);
+ }
+
+ public static boolean CheckDirExists(String dir) {
+ File file = new File(dir);
+ return file.isDirectory();
}
public static long nimbusVersionOfBlob(String key, ClientBlobStore cb) throws AuthorizationException, KeyNotFoundException {
@@ -626,27 +697,19 @@ public class Utils {
}
- public static synchronized IFn loadClojureFn(String namespace, String name) {
+ public static synchronized clojure.lang.IFn loadClojureFn(String namespace, String name) {
try {
clojure.lang.Compiler.eval(RT.readString("(require '" + namespace + ")"));
} catch (Exception e) {
//if playing from the repl and defining functions, file won't exist
}
- return (IFn) RT.var(namespace, name).deref();
+ return (clojure.lang.IFn) RT.var(namespace, name).deref();
}
public static boolean isSystemId(String id) {
return id.startsWith("__");
}
- public static <K, V> Map<V, K> reverseMap(Map<K, V> map) {
- Map<V, K> ret = new HashMap<V, K>();
- for (Map.Entry<K, V> entry : map.entrySet()) {
- ret.put(entry.getValue(), entry.getKey());
- }
- return ret;
- }
-
public static ComponentCommon getComponentCommon(StormTopology topology, String id) {
if (topology.get_spouts().containsKey(id)) {
return topology.get_spouts().get(id).get_common();
@@ -714,7 +777,7 @@ public class Utils {
}
}
- public static <T> T thriftDeserialize(Class c, byte[] b) {
+ public static <T> T thriftDeserialize(Class<T> c, byte[] b) {
try {
return Utils.thriftDeserialize(c, b, 0, b.length);
} catch (Exception e) {
@@ -874,7 +937,7 @@ public class Utils {
}
boolean gzipped = inFile.toString().endsWith("gz");
- if (onWindows()) {
+ if (isOnWindows()) {
// Tar is not native to Windows. Use simple Java based implementation for
// tests and simple tar archives
unTarUsingJava(inFile, untarDir, gzipped);
@@ -916,7 +979,6 @@ public class Utils {
private static void unTarUsingJava(File inFile, File untarDir,
boolean gzipped) throws IOException {
InputStream inputStream = null;
- TarArchiveInputStream tis = null;
try {
if (gzipped) {
inputStream = new BufferedInputStream(new GZIPInputStream(
@@ -924,31 +986,15 @@ public class Utils {
} else {
inputStream = new BufferedInputStream(new FileInputStream(inFile));
}
- tis = new TarArchiveInputStream(inputStream);
- for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null; ) {
- unpackEntries(tis, entry, untarDir);
- entry = tis.getNextTarEntry();
+ try (TarArchiveInputStream tis = new TarArchiveInputStream(inputStream)) {
+ for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null; ) {
+ unpackEntries(tis, entry, untarDir);
+ entry = tis.getNextTarEntry();
+ }
}
} finally {
- cleanup(tis, inputStream);
- }
- }
-
- /**
- * Close the Closeable objects and <b>ignore</b> any {@link IOException} or
- * null pointers. Must only be used for cleanup in exception handlers.
- *
- * @param closeables the objects to close
- */
- private static void cleanup(java.io.Closeable... closeables) {
- for (java.io.Closeable c : closeables) {
- if (c != null) {
- try {
- c.close();
- } catch (IOException e) {
- LOG.debug("Exception in closing " + c, e);
-
- }
+ if(inputStream != null) {
+ inputStream.close();
}
}
}
@@ -970,7 +1016,7 @@ public class Utils {
if (!outputFile.getParentFile().exists()) {
if (!outputFile.getParentFile().mkdirs()) {
throw new IOException("Mkdirs failed to create tar internal dir "
- + outputDir);
+ + outputDir);
}
}
int count;
@@ -985,13 +1031,17 @@ public class Utils {
outputStream.close();
}
- public static boolean onWindows() {
+ public static boolean isOnWindows() {
if (System.getenv("OS") != null) {
return System.getenv("OS").equals("Windows_NT");
}
return false;
}
+ public static boolean isAbsolutePath(String path) {
+ return Paths.get(path).isAbsolute();
+ }
+
public static void unpack(File localrsrc, File dst) throws IOException {
String lowerDst = localrsrc.getName().toLowerCase();
if (lowerDst.endsWith(".jar")) {
@@ -1031,6 +1081,10 @@ public class Utils {
return newCurator(conf, servers, port, root, null);
}
+ public static CuratorFramework newCurator(Map conf, List<String> servers, Object port, ZookeeperAuthInfo auth) {
+ return newCurator(conf, servers, port, "", auth);
+ }
+
public static CuratorFramework newCurator(Map conf, List<String> servers, Object port, String root, ZookeeperAuthInfo auth) {
List<String> serverPorts = new ArrayList<String>();
for (String zkServer : servers) {
@@ -1071,17 +1125,19 @@ public class Utils {
.connectionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT)))
.sessionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)))
.retryPolicy(new StormBoundedExponentialBackoffRetry(
- Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)),
- Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)),
- Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES))));
+ Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)),
+ Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)),
+ Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES))));
if (auth != null && auth.scheme != null && auth.payload != null) {
builder.authorization(auth.scheme, auth.payload);
}
}
- public static CuratorFramework newCurator(Map conf, List<String> servers, Object port, ZookeeperAuthInfo auth) {
- return newCurator(conf, servers, port, "", auth);
+ public static void testSetupBuilder(CuratorFrameworkFactory.Builder
+ builder, String zkStr, Map conf, ZookeeperAuthInfo auth)
+ {
+ setupBuilder(builder, zkStr, conf, auth);
}
public static CuratorFramework newCuratorStarted(Map conf, List<String> servers, Object port, String root, ZookeeperAuthInfo auth) {
@@ -1122,10 +1178,16 @@ public class Utils {
LOG.info("{}:{}", prefix, line);
}
} catch (IOException e) {
- LOG.warn("Error whiel trying to log stream", e);
+ LOG.warn("Error while trying to log stream", e);
}
}
+ /**
+ * Checks if a throwable is an instance of a particular class
+ * @param klass The class you're expecting
+ * @param throwable The throwable you expect to be an instance of klass
+ * @return true if throwable is instance of klass, false otherwise.
+ */
public static boolean exceptionCauseIsInstanceOf(Class klass, Throwable throwable) {
Throwable t = throwable;
while (t != null) {
@@ -1161,6 +1223,7 @@ public class Utils {
&& !((String)conf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME)).isEmpty());
}
+
public static List<ACL> getWorkerACL(Map conf) {
//This is a work around to an issue with ZK where a sasl super user is not super unless there is an open SASL ACL so we are trying to give the correct perms
if (!isZkAuthenticationConfiguredTopology(conf)) {
@@ -1168,11 +1231,11 @@ public class Utils {
}
String stormZKUser = (String)conf.get(Config.STORM_ZOOKEEPER_SUPERACL);
if (stormZKUser == null) {
- throw new IllegalArgumentException("Authentication is enabled but "+Config.STORM_ZOOKEEPER_SUPERACL+" is not set");
+ throw new IllegalArgumentException("Authentication is enabled but " + Config.STORM_ZOOKEEPER_SUPERACL + " is not set");
}
- String[] split = stormZKUser.split(":",2);
+ String[] split = stormZKUser.split(":", 2);
if (split.length != 2) {
- throw new IllegalArgumentException(Config.STORM_ZOOKEEPER_SUPERACL+" does not appear to be in the form scheme:acl, i.e. sasl:storm-user");
+ throw new IllegalArgumentException(Config.STORM_ZOOKEEPER_SUPERACL + " does not appear to be in the form scheme:acl, i.e. sasl:storm-user");
}
ArrayList<ACL> ret = new ArrayList<ACL>(ZooDefs.Ids.CREATOR_ALL_ACL);
ret.add(new ACL(ZooDefs.Perms.ALL, new Id(split[0], split[1])));
@@ -1211,6 +1274,10 @@ public class Utils {
}
}
+ /**
+ * Gets some information, including stack trace, for a running thread.
+ * @return A human-readable string of the dump.
+ */
public static String threadDump() {
final StringBuilder dump = new StringBuilder();
final java.lang.management.ThreadMXBean threadMXBean = java.lang.management.ManagementFactory.getThreadMXBean();
@@ -1236,20 +1303,19 @@ public class Utils {
return dump.toString();
}
- // Assumes caller is synchronizing
+ /**
+ * Creates an instance of the pluggable SerializationDelegate or falls back to
+ * DefaultSerializationDelegate if something goes wrong.
+ * @param stormConf The config from which to pull the name of the pluggable class.
+ * @return an instance of the class specified by storm.meta.serialization.delegate
+ */
private static SerializationDelegate getSerializationDelegate(Map stormConf) {
String delegateClassName = (String)stormConf.get(Config.STORM_META_SERIALIZATION_DELEGATE);
SerializationDelegate delegate;
try {
Class delegateClass = Class.forName(delegateClassName);
delegate = (SerializationDelegate) delegateClass.newInstance();
- } catch (ClassNotFoundException e) {
- LOG.error("Failed to construct serialization delegate, falling back to default", e);
- delegate = new DefaultSerializationDelegate();
- } catch (InstantiationException e) {
- LOG.error("Failed to construct serialization delegate, falling back to default", e);
- delegate = new DefaultSerializationDelegate();
- } catch (IllegalAccessException e) {
+ } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) {
LOG.error("Failed to construct serialization delegate, falling back to default", e);
delegate = new DefaultSerializationDelegate();
}
@@ -1295,7 +1361,7 @@ public class Utils {
if (!file.getParentFile().mkdirs()) {
if (!file.getParentFile().isDirectory()) {
throw new IOException("Mkdirs failed to create " +
- file.getParentFile().toString());
+ file.getParentFile().toString());
}
}
OutputStream out = new FileOutputStream(file);
@@ -1358,13 +1424,16 @@ public class Utils {
}
if (memoryOpts != null) {
int unit = 1;
- if (memoryOpts.toLowerCase().endsWith("k")) {
+ memoryOpts = memoryOpts.toLowerCase();
+
+ if (memoryOpts.endsWith("k")) {
unit = 1024;
- } else if (memoryOpts.toLowerCase().endsWith("m")) {
+ } else if (memoryOpts.endsWith("m")) {
unit = 1024 * 1024;
- } else if (memoryOpts.toLowerCase().endsWith("g")) {
+ } else if (memoryOpts.endsWith("g")) {
unit = 1024 * 1024 * 1024;
}
+
memoryOpts = memoryOpts.replaceAll("[a-zA-Z]", "");
Double result = Double.parseDouble(memoryOpts) * unit / 1024.0 / 1024.0;
return (result < 1.0) ? 1.0 : result;
@@ -1387,18 +1456,29 @@ public class Utils {
}
public static TopologyInfo getTopologyInfo(String name, String asUser, Map stormConf) {
- TopologyInfo topologyInfo = null;
try (NimbusClient client = NimbusClient.getConfiguredClientAs(stormConf, asUser)) {
- ClusterSummary summary = client.getClient().getClusterInfo();
+ String topologyId = getTopologyId(name, client.getClient());
+ if (null != topologyId) {
+ return client.getClient().getTopologyInfo(topologyId);
+ }
+ return null;
+ } catch(Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static String getTopologyId(String name, Nimbus.Client client) {
+ try {
+ ClusterSummary summary = client.getClusterInfo();
for(TopologySummary s : summary.get_topologies()) {
if(s.get_name().equals(name)) {
- topologyInfo = client.getClient().getTopologyInfo(s.get_id());
+ return s.get_id();
}
}
} catch(Exception e) {
throw new RuntimeException(e);
}
- return topologyInfo;
+ return null;
}
/**
@@ -1414,8 +1494,744 @@ public class Utils {
return number & Integer.MAX_VALUE;
}
- public static String localHostname() throws UnknownHostException {
+ public static GlobalStreamId getGlobalStreamId(String streamId, String componentId) {
+ if (componentId == null) {
+ return new GlobalStreamId(streamId, DEFAULT_STREAM_ID);
+ }
+ return new GlobalStreamId(streamId, componentId);
+ }
+
+ public static RuntimeException wrapInRuntime(Exception e){
+ if (e instanceof RuntimeException){
+ return (RuntimeException)e;
+ } else {
+ return new RuntimeException(e);
+ }
+ }
+
+ public static int getAvailablePort(int prefferedPort) {
+ int localPort = -1;
+ try(ServerSocket socket = new ServerSocket(prefferedPort)) {
+ localPort = socket.getLocalPort();
+ } catch(IOException exp) {
+ if (prefferedPort > 0) {
+ return getAvailablePort(0);
+ }
+ }
+ return localPort;
+ }
+
+ public static int getAvailablePort() {
+ return getAvailablePort(0);
+ }
+
+ /**
+ * Determines if a zip archive contains a particular directory.
+ *
+ * @param zipfile path to the zipped file
+ * @param target directory being looked for in the zip.
+ * @return boolean whether or not the directory exists in the zip.
+ */
+ public static boolean zipDoesContainDir(String zipfile, String target) throws IOException {
+ List<ZipEntry> entries = (List<ZipEntry>) Collections.list(new ZipFile(zipfile).entries());
+
+ String targetDir = target + "/";
+ for(ZipEntry entry : entries) {
+ String name = entry.getName();
+ if(name.startsWith(targetDir)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Joins any number of maps together into a single map, combining their values into
+ * a list, maintaining values in the order the maps were passed in. Nulls are inserted
+ * for given keys when the map does not contain that key.
+ *
+ * i.e. joinMaps({'a' => 1, 'b' => 2}, {'b' => 3}, {'a' => 4, 'c' => 5}) ->
+ * {'a' => [1, null, 4], 'b' => [2, 3, null], 'c' => [null, null, 5]}
+ *
+ * @param maps variable number of maps to join - order affects order of values in output.
+ * @return combined map
+ */
+ public static <K, V> Map<K, List<V>> joinMaps(Map<K, V>... maps) {
+ Map<K, List<V>> ret = new HashMap<>();
+
+ Set<K> keys = new HashSet<>();
+
+ for(Map<K, V> map : maps) {
+ keys.addAll(map.keySet());
+ }
+
+ for(Map<K, V> m : maps) {
+ for(K key : keys) {
+ V value = m.get(key);
+
+ if(!ret.containsKey(key)) {
+ ret.put(key, new ArrayList<V>());
+ }
+
+ List<V> targetList = ret.get(key);
+ targetList.add(value);
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Fills up chunks out of a collection (given a maximum amount of chunks)
+ *
+ * i.e. partitionFixed(5, [1,2,3]) -> [[1,2,3]]
+ * partitionFixed(5, [1..9]) -> [[1,2], [3,4], [5,6], [7,8], [9]]
+ * partitionFixed(3, [1..10]) -> [[1,2,3,4], [5,6,7], [8,9,10]]
+ * @param maxNumChunks the maximum number of chunks to return
+ * @param coll the collection to be chunked up
+ * @return a list of the chunks, which are themselves lists.
+ */
+ public static <T> List<List<T>> partitionFixed(int maxNumChunks, Collection<T> coll) {
+ List<List<T>> ret = new ArrayList<>();
+
+ if(maxNumChunks == 0 || coll == null) {
+ return ret;
+ }
+
+ Map<Integer, Integer> parts = integerDivided(coll.size(), maxNumChunks);
+
+ // Keys sorted in descending order
+ List<Integer> sortedKeys = new ArrayList<Integer>(parts.keySet());
+ Collections.sort(sortedKeys, Collections.reverseOrder());
+
+
+ Iterator<T> it = coll.iterator();
+ for(Integer chunkSize : sortedKeys) {
+ if(!it.hasNext()) { break; }
+ Integer times = parts.get(chunkSize);
+ for(int i = 0; i < times; i++) {
+ if(!it.hasNext()) { break; }
+ List<T> chunkList = new ArrayList<>();
+ for(int j = 0; j < chunkSize; j++) {
+ if(!it.hasNext()) { break; }
+ chunkList.add(it.next());
+ }
+ ret.add(chunkList);
+ }
+ }
+
+ return ret;
+ }
+
+ /**
+ * Return a new instance of a pluggable specified in the conf.
+ * @param conf The conf to read from.
+ * @param configKey The key pointing to the pluggable class
+ * @return an instance of the class or null if it is not specified.
+ */
+ public static Object getConfiguredClass(Map conf, Object configKey) {
+ if (conf.containsKey(configKey)) {
+ return newInstance((String)conf.get(configKey));
+ }
+ return null;
+ }
+
+ public static String logsFilename(String stormId, String port) {
+ return stormId + FILE_PATH_SEPARATOR + port + FILE_PATH_SEPARATOR + "worker.log";
+ }
+
+ public static String eventLogsFilename(String stormId, String port) {
+ return stormId + FILE_PATH_SEPARATOR + port + FILE_PATH_SEPARATOR + "events.log";
+ }
+
+ public static Object readYamlFile(String yamlFile) {
+ try (FileReader reader = new FileReader(yamlFile)) {
+ return new Yaml(new SafeConstructor()).load(reader);
+ } catch(Exception ex) {
+ LOG.error("Failed to read yaml file.", ex);
+ }
+ return null;
+ }
+
+ public static void setupDefaultUncaughtExceptionHandler() {
+ Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ public void uncaughtException(Thread thread, Throwable thrown) {
+ try {
+ handleUncaughtException(thrown);
+ } catch (Error err) {
+ LOG.error("Received error in main thread.. terminating server...", err);
+ Runtime.getRuntime().exit(-2);
+ }
+ }
+ });
+ }
+
+ /**
+ * Creates a new map with a string value in the map replaced with an
+ * equivalently-lengthed string of '#'.
+ * @param m The map that a value will be redacted from
+ * @param key The key pointing to the value to be redacted
+ * @return a new map with the value redacted. The original map will not be modified.
+ */
+ public static Map<Object, String> redactValue(Map<Object, String> m, Object key) {
+ if(m.containsKey(key)) {
+ HashMap<Object, String> newMap = new HashMap<>(m);
+ String value = newMap.get(key);
+ String redacted = new String(new char[value.length()]).replace("\0", "#");
+ newMap.put(key, redacted);
+ return newMap;
+ }
+ return m;
+ }
+
+ /**
+ * Make sure a given key name is valid for the storm config.
+ * Throw RuntimeException if the key isn't valid.
+ * @param name The name of the config key to check.
+ */
+ private static final Set<String> disallowedKeys = new HashSet<>(Arrays.asList(new String[] {"/", ".", ":", "\\"}));
+ public static void validateKeyName(String name) {
+
+ for(String key : disallowedKeys) {
+ if( name.contains(key) ) {
+ throw new RuntimeException("Key name cannot contain any of the following: " + disallowedKeys.toString());
+ }
+ }
+ if(name.trim().isEmpty()) {
+ throw new RuntimeException("Key name cannot be blank");
+ }
+ }
+
+ public static String localHostname () throws UnknownHostException {
+ return _instance.localHostnameImpl();
+ }
+
+ // Non-static impl methods exist for mocking purposes.
+ protected String localHostnameImpl () throws UnknownHostException {
return InetAddress.getLocalHost().getCanonicalHostName();
}
-}
+ private static String memoizedLocalHostnameString = null;
+
+ public static String memoizedLocalHostname () throws UnknownHostException {
+ if (memoizedLocalHostnameString == null) {
+ memoizedLocalHostnameString = localHostname();
+ }
+ return memoizedLocalHostnameString;
+ }
+
+ /**
+ * Gets the storm.local.hostname value, or tries to figure out the local hostname
+ * if it is not set in the config.
+ * @param conf The storm config to read from
+ * @return a string representation of the hostname.
+ */
+ public static String hostname (Map<String, Object> conf) throws UnknownHostException {
+ if (conf == null) {
+ return memoizedLocalHostname();
+ }
+ Object hostnameString = conf.get(Config.STORM_LOCAL_HOSTNAME);
+ if (hostnameString == null || hostnameString.equals("")) {
+ return memoizedLocalHostname();
+ }
+ return (String)hostnameString;
+ }
+
+ public static String uuid() {
+ return UUID.randomUUID().toString();
+ }
+
+ public static void exitProcess (int val, String msg) {
+ String combinedErrorMessage = "Halting process: " + msg;
+ LOG.error(combinedErrorMessage, new RuntimeException(combinedErrorMessage));
+ Runtime.getRuntime().exit(val);
+ }
+
+ public static Runnable mkSuicideFn() {
+ return new Runnable() {
+ @Override
+ public void run() {
+ Utils.exitProcess(1, "Worker died");
+ }
+ };
+ }
+
+ /**
+ * "{:a 1 :b 1 :c 2} -> {1 [:a :b] 2 :c}"
+ *
+ * Example usage in java:
+ * Map<Integer, String> tasks;
+ * Map<String, List<Integer>> componentTasks = Utils.reverse_map(tasks);
+ *
+ * The order of he resulting list values depends on the ordering properties
+ * of the Map passed in. The caller is responsible for passing an ordered
+ * map if they expect the result to be consistently ordered as well.
+ *
+ * @param map to reverse
+ * @return a reversed map
+ */
+ public static <K, V> HashMap<V, List<K>> reverseMap(Map<K, V> map) {
+ HashMap<V, List<K>> rtn = new HashMap<V, List<K>>();
+ if (map == null) {
+ return rtn;
+ }
+ for (Entry<K, V> entry : map.entrySet()) {
+ K key = entry.getKey();
+ V val = entry.getValue();
+ List<K> list = rtn.get(val);
+ if (list == null) {
+ list = new ArrayList<K>();
+ rtn.put(entry.getValue(), list);
+ }
+ list.add(key);
+ }
+ return rtn;
+ }
+
+ /**
+ * "[[:a 1] [:b 1] [:c 2]} -> {1 [:a :b] 2 :c}"
+ * Reverses an assoc-list style Map like reverseMap(Map...)
+ *
+ * @param listSeq to reverse
+ * @return a reversed map
+ */
+ public static HashMap reverseMap(List listSeq) {
+ HashMap<Object, List<Object>> rtn = new HashMap();
+ if (listSeq == null) {
+ return rtn;
+ }
+ for (Object entry : listSeq) {
+ List listEntry = (List) entry;
+ Object key = listEntry.get(0);
+ Object val = listEntry.get(1);
+ List list = rtn.get(val);
+ if (list == null) {
+ list = new ArrayList<Object>();
+ rtn.put(val, list);
+ }
+ list.add(key);
+ }
+ return rtn;
+ }
+
+
+ /**
+ * @return the pid of this JVM, because Java doesn't provide a real way to do this.
+ */
+ public static String processPid() {
+ String name = ManagementFactory.getRuntimeMXBean().getName();
+ String[] split = name.split("@");
+ if (split.length != 2) {
+ throw new RuntimeException("Got unexpected process name: " + name);
+ }
+ return split[0];
+ }
+
+ public static int execCommand(String... command) throws ExecuteException, IOException {
+ CommandLine cmd = new CommandLine(command[0]);
+ for (int i = 1; i < command.length; i++) {
+ cmd.addArgument(command[i]);
+ }
+
+ DefaultExecutor exec = new DefaultExecutor();
+ return exec.execute(cmd);
+ }
+
+ /**
+ * Extract dir from the jar to destdir
+ *
+ * @param jarpath Path to the jar file
+ * @param dir Directory in the jar to pull out
+ * @param destdir Path to the directory where the extracted directory will be put
+ */
+ public static void extractDirFromJar(String jarpath, String dir, File destdir) {
+ _instance.extractDirFromJarImpl(jarpath, dir, destdir);
+ }
+
+ public void extractDirFromJarImpl(String jarpath, String dir, File destdir) {
+ try (JarFile jarFile = new JarFile(jarpath)) {
+ Enumeration<JarEntry> jarEnums = jarFile.entries();
+ while (jarEnums.hasMoreElements()) {
+ JarEntry entry = jarEnums.nextElement();
+ if (!entry.isDirectory() && entry.getName().startsWith(dir)) {
+ File aFile = new File(destdir, entry.getName());
+ aFile.getParentFile().mkdirs();
+ try (FileOutputStream out = new FileOutputStream(aFile);
+ InputStream in = jarFile.getInputStream(entry)) {
+ IOUtils.copy(in, out);
+ }
+ }
+ }
+ } catch (IOException e) {
+ LOG.info("Could not extract {} from {}", dir, jarpath);
+ }
+ }
+
+ public static void sendSignalToProcess(long lpid, int signum) throws IOException {
+ String pid = Long.toString(lpid);
+ try {
+ if (isOnWindows()) {
+ if (signum == SIGKILL) {
+ execCommand("taskkill", "/f", "/pid", pid);
+ } else {
+ execCommand("taskkill", "/pid", pid);
+ }
+ } else {
+ execCommand("kill", "-" + signum, pid);
+ }
+ } catch (ExecuteException e) {
+ LOG.info("Error when trying to kill {}. Process is probably already dead.", pid);
+ } catch (IOException e) {
+ LOG.info("IOException Error when trying to kill {}.", pid);
+ throw e;
+ }
+ }
+
+ public static void forceKillProcess (String pid) throws IOException {
+ sendSignalToProcess(Long.parseLong(pid), SIGKILL);
+ }
+
+ public static void killProcessWithSigTerm (String pid) throws IOException {
+ sendSignalToProcess(Long.parseLong(pid), SIGTERM);
+ }
+
+ /**
+ * Adds the user supplied function as a shutdown hook for cleanup.
+ * Also adds a function that sleeps for a second and then halts the
+ * runtime to avoid any zombie process in case cleanup function hangs.
+ */
+ public static void addShutdownHookWithForceKillIn1Sec (Runnable func) {
+ Runnable sleepKill = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ Time.sleepSecs(1);
+ Runtime.getRuntime().halt(20);
+ } catch (Exception e) {
+ LOG.warn("Exception in the ShutDownHook", e);
+ }
+ }
+ };
+ Runtime.getRuntime().addShutdownHook(new Thread(func));
+ Runtime.getRuntime().addShutdownHook(new Thread(sleepKill));
+ }
+
+ /**
+ * Returns the combined string, escaped for posix shell.
+ * @param command the list of strings to be combined
+ * @return the resulting command string
+ */
+ public static String shellCmd (List<String> command) {
+ List<String> changedCommands = new ArrayList<>(command.size());
+ for (String str: command) {
+ if (str == null) {
+ continue;
+ }
+ changedCommands.add("'" + str.replaceAll("'", "'\"'\"'") + "'");
+ }
+ return StringUtils.join(changedCommands, " ");
+ }
+
+ public static String scriptFilePath (String dir) {
+ return dir + FILE_PATH_SEPARATOR + "storm-worker-script.sh";
+ }
+
+ public static String containerFilePath (String dir) {
+ return dir + FILE_PATH_SEPARATOR + "launch_container.sh";
+ }
+
+ public static Object nullToZero (Object v) {
+ return (v != null ? v : 0);
+ }
+
+ /**
+ * Deletes a file or directory and its contents if it exists. Does not
+ * complain if the input is null or does not exist.
+ * @param path the path to the file or directory
+ */
+ public static void forceDelete(String path) throws IOException {
+ _instance.forceDeleteImpl(path);
+ }
+
+ // Non-static impl methods exist for mocking purposes.
+ protected void forceDeleteImpl(String path) throws IOException {
+ LOG.debug("Deleting path {}", path);
+ if (checkFileExists(path)) {
+ try {
+ FileUtils.forceDelete(new File(path));
+ } catch (FileNotFoundException ignored) {}
+ }
+ }
+
+ /**
+ * Returns a Collection of file names found under the given directory.
+ * @param dir a directory
+ * @return the Collection of file names
+ */
+ public static Collection<String> readDirContents(String dir) {
+ Collection<String> ret = new HashSet<>();
+ File[] files = new File(dir).listFiles();
+ if (files != null) {
+ for (File f: files) {
+ ret.add(f.getName());
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Returns the value of java.class.path System property. Kept separate for
+ * testing.
+ * @return the classpath
+ */
+ public static String currentClasspath() {
+ return _instance.currentClasspathImpl();
+ }
+
+ // Non-static impl methods exist for mocking purposes.
+ public String currentClasspathImpl() {
+ return System.getProperty("java.class.path");
+ }
+
+ public static String addToClasspath(String classpath,
+ Collection<String> paths) {
+ return _instance.addToClasspathImpl(classpath, paths);
+ }
+
+ public static String addToClasspath(Collection<String> classpaths,
+ Collection<String> paths) {
+ return _instance.addToClasspathImpl(classpaths, paths);
+ }
+
+ // Non-static impl methods exist for mocking purposes.
+ public String addToClasspathImpl(String classpath,
+ Collection<String> paths) {
+ if (paths == null || paths.isEmpty()) {
+ return classpath;
+ }
+ List<String> l = new LinkedList<>();
+ l.add(classpath);
+ l.addAll(paths);
+ return StringUtils.join(l, CLASS_PATH_SEPARATOR);
+ }
+
+ public String addToClasspathImpl(Collection<String> classpaths,
+ Collection<String> paths) {
+ List<String> allPaths = new ArrayList<>();
+ if(classpaths != null) {
+ allPaths.addAll(classpaths);
+ }
+ if(paths != null) {
+ allPaths.addAll(paths);
+ }
+ return StringUtils.join(allPaths, CLASS_PATH_SEPARATOR);
+ }
+
+ public static class UptimeComputer {
+ int startTime = 0;
+
+ public UptimeComputer() {
+ startTime = Time.currentTimeSecs();
+ }
+
+ public int upTime() {
+ return Time.deltaSecs(startTime);
+ }
+ }
+
+ public static UptimeComputer makeUptimeComputer() {
+ return _instance.makeUptimeComputerImpl();
+ }
+
+ // Non-static impl methods exist for mocking purposes.
+ public UptimeComputer makeUptimeComputerImpl() {
+ return new UptimeComputer();
+ }
+
+ /**
+ * Writes a posix shell script file to be executed in its own process.
+ * @param dir the directory under which the script is to be written
+ * @param command the command the script is to execute
+ * @param environment optional environment variables to set before running the script's command. May be null.
+ * @return the path to the script that has been written
+ */
+ public static String writeScript(String dir, List<String> command,
+ Map<String,String> environment) throws IOException {
+ String path = Utils.scriptFilePath(dir);
+ try(BufferedWriter out = new BufferedWriter(new FileWriter(path))) {
+ out.write("#!/bin/bash");
+ out.newLine();
+ if (environment != null) {
+ for (String k : environment.keySet()) {
+ String v = environment.get(k);
+ if (v == null) {
+ v = "";
+ }
+ out.write(Utils.shellCmd(
+ Arrays.asList(
+ "export",k+"="+v)));
+ out.write(";");
+ out.newLine();
+ }
+ }
+ out.newLine();
+ out.write("exec "+Utils.shellCmd(command)+";");
+ }
+ return path;
+ }
+
+ /**
+ * A thread that can answer if it is sleeping in the case of simulated time.
+ * This class is not useful when simulated time is not being used.
+ */
+ public static class SmartThread extends Thread {
+ public boolean isSleeping() {
+ return Time.isThreadWaiting(this);
+ }
+ public SmartThread(Runnable r) {
+ super(r);
+ }
+ }
+
+ /**
+ * Creates a thread that calls the given code repeatedly, sleeping for an
+ * interval of seconds equal to the return value of the previous call.
+ *
+ * The given afn may be a callable that returns the number of seconds to
+ * sleep, or it may be a Callable that returns another Callable that in turn
+ * returns the number of seconds to sleep. In the latter case isFactory.
+ *
+ * @param afn the code to call on each iteration
+ * @param isDaemon whether the new thread should be a daemon thread
+ * @param eh code to call when afn throws an exception
+ * @param priority the new thread's priority
+ * @param isFactory whether afn returns a callable instead of sleep seconds
+ * @param startImmediately whether to start the thread before returning
+ * @param threadName a suffix to be appended to the thread name
+ * @return the newly created thread
+ * @see java.lang.Thread
+ */
+ public static SmartThread asyncLoop(final Callable afn,
+ boolean isDaemon, final Thread.UncaughtExceptionHandler eh,
+ int priority, final boolean isFactory, boolean startImmediately,
+ String threadName) {
+ SmartThread thread = new SmartThread(new Runnable() {
+ public void run() {
+ Object s;
+ try {
+ Callable fn = isFactory ? (Callable) afn.call() : afn;
+ while ((s = fn.call()) instanceof Long) {
+ Time.sleepSecs((Long) s);
+ }
+ } catch (Throwable t) {
+ if (Utils.exceptionCauseIsInstanceOf(
+ InterruptedException.class, t)) {
+ LOG.info("Async loop interrupted!");
+ return;
+ }
+ LOG.error("Async loop died!", t);
+ throw new RuntimeException(t);
+ }
+ }
+ });
+ if (eh != null) {
+ thread.setUncaughtExceptionHandler(eh);
+ } else {
+ thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ public void uncaughtException(Thread t, Throwable e) {
+ LOG.error("Async loop died!", e);
+ Utils.exitProcess(1, "Async loop died!");
+ }
+ });
+ }
+ thread.setDaemon(isDaemon);
+ thread.setPriority(priority);
+ if (threadName != null && !threadName.isEmpty()) {
+ thread.setName(thread.getName() +"-"+ threadName);
+ }
+ if (startImmediately) {
+ thread.start();
+ }
+ return thread;
+ }
+
+ /**
+ * Convenience method used when only the function and name suffix are given.
+ * @param afn the code to call on each iteration
+ * @param threadName a suffix to be appended to the thread name
+ * @return the newly created thread
+ * @see java.lang.Thread
+ */
+ public static SmartThread asyncLoop(final Callable afn, String threadName, final Thread.UncaughtExceptionHandler eh) {
+ return asyncLoop(afn, false, eh, Thread.NORM_PRIORITY, false, true,
+ threadName);
+ }
+
+ /**
+ * Convenience method used when only the function is given.
+ * @param afn the code to call on each iteration
+ * @return the newly created thread
+ */
+ public static SmartThread asyncLoop(final Callable afn) {
+ return asyncLoop(afn, false, null, Thread.NORM_PRIORITY, false, true,
+ null);
+ }
+
+ public static <T> List<T> interleaveAll(List<List<T>> nodeList) {
+ if (nodeList != null && nodeList.size() > 0) {
+ List<T> first = new ArrayList<T>();
+ List<List<T>> rest = new ArrayList<List<T>>();
+ for (List<T> node : nodeList) {
+ if (node != null && node.size() > 0) {
+ first.add(node.get(0));
+ rest.add(node.subList(1, node.size()));
+ }
+ }
+ List<T> interleaveRest = interleaveAll(rest);
+ if (interleaveRest != null) {
+ first.addAll(interleaveRest);
+ }
+ return first;
+ }
+ return null;
+ }
+
+ public static long bitXor(Long a, Long b) {
+ return a ^ b;
+ }
+
+ public static List<String> getRepeat(List<String> list) {
+ List<String> rtn = new ArrayList<String>();
+ Set<String> idSet = new HashSet<String>();
+
+ for (String id : list) {
+ if (idSet.contains(id)) {
+ rtn.add(id);
+ } else {
+ idSet.add(id);
+ }
+ }
+
+ return rtn;
+ }
+
+ /**
+ * converts a clojure PersistentMap to java HashMap
+ */
+ public static Map<String, Object> convertClojureMapToJavaMap(Map map) {
+ Map<String, Object> ret = new HashMap<>(map.size());
+ for (Object obj : map.entrySet()) {
+ Map.Entry entry = (Map.Entry) obj;
+ Keyword keyword = (Keyword) entry.getKey();
+ String key = keyword.getName();
+ if (key.startsWith(":")) {
+ key = key.substring(1, key.length());
+ }
+ Object value = entry.getValue();
+ ret.put(key, value);
+ }
+
+ return ret;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/zookeeper/LeaderElectorImp.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/zookeeper/LeaderElectorImp.java b/storm-core/src/jvm/org/apache/storm/zookeeper/LeaderElectorImp.java
new file mode 100644
index 0000000..74816c2
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/zookeeper/LeaderElectorImp.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.zookeeper;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.leader.LeaderLatch;
+import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
+import org.apache.curator.framework.recipes.leader.Participant;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.nimbus.ILeaderElector;
+import org.apache.storm.nimbus.NimbusInfo;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class LeaderElectorImp implements ILeaderElector {
+ private static Logger LOG = LoggerFactory.getLogger(LeaderElectorImp.class);
+ private final Map conf;
+ private final List<String> servers;
+ private final CuratorFramework zk;
+ private final String leaderlockPath;
+ private final String id;
+ private final AtomicReference<LeaderLatch> leaderLatch;
+ private final AtomicReference<LeaderLatchListener> leaderLatchListener;
+ private final BlobStore blobStore;
+
+ public LeaderElectorImp(Map conf, List<String> servers, CuratorFramework zk, String leaderlockPath, String id, AtomicReference<LeaderLatch> leaderLatch,
+ AtomicReference<LeaderLatchListener> leaderLatchListener, BlobStore blobStore) {
+ this.conf = conf;
+ this.servers = servers;
+ this.zk = zk;
+ this.leaderlockPath = leaderlockPath;
+ this.id = id;
+ this.leaderLatch = leaderLatch;
+ this.leaderLatchListener = leaderLatchListener;
+ this.blobStore = blobStore;
+ }
+
+ @Override
+ public void prepare(Map conf) {
+ // no-op for zookeeper implementation
+ }
+
+ @Override
+ public void addToLeaderLockQueue() throws Exception {
+ // if this latch is already closed, we need to create new instance.
+ if (LeaderLatch.State.CLOSED.equals(leaderLatch.get().getState())) {
+ leaderLatch.set(new LeaderLatch(zk, leaderlockPath));
+ leaderLatchListener.set(Zookeeper.leaderLatchListenerImpl(conf, zk, blobStore, leaderLatch.get()));
+ LOG.info("LeaderLatch was in closed state. Resetted the leaderLatch and listeners.");
+ }
+ // Only if the latch is not already started we invoke start
+ if (LeaderLatch.State.LATENT.equals(leaderLatch.get().getState())) {
+ leaderLatch.get().addListener(leaderLatchListener.get());
+ leaderLatch.get().start();
+ LOG.info("Queued up for leader lock.");
+ } else {
+ LOG.info("Node already in queue for leader lock.");
+ }
+ }
+
+ @Override
+ // Only started latches can be closed.
+ public void removeFromLeaderLockQueue() throws Exception {
+ if (LeaderLatch.State.STARTED.equals(leaderLatch.get().getState())) {
+ leaderLatch.get().close();
+ LOG.info("Removed from leader lock queue.");
+ } else {
+ LOG.info("leader latch is not started so no removeFromLeaderLockQueue needed.");
+ }
+ }
+
+ @Override
+ public boolean isLeader() throws Exception {
+ return leaderLatch.get().hasLeadership();
+ }
+
+ @Override
+ public NimbusInfo getLeader() {
+ try {
+ return Zookeeper.toNimbusInfo(leaderLatch.get().getLeader());
+ } catch (Exception e) {
+ throw Utils.wrapInRuntime(e);
+ }
+ }
+
+ @Override
+ public List<NimbusInfo> getAllNimbuses() throws Exception {
+ List<NimbusInfo> nimbusInfos = new ArrayList<>();
+ Collection<Participant> participants = leaderLatch.get().getParticipants();
+ for (Participant participant : participants) {
+ nimbusInfos.add(Zookeeper.toNimbusInfo(participant));
+ }
+ return nimbusInfos;
+ }
+
+ @Override
+ public void close() {
+ LOG.info("closing zookeeper connection of leader elector.");
+ zk.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java b/storm-core/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java
new file mode 100644
index 0000000..5247558
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.zookeeper;
+
+import org.apache.zookeeper.Watcher;
+
+import java.util.HashMap;
+
+public class ZkEventTypes {
+
+ private static HashMap<Watcher.Event.EventType, String> map;
+
+ static {
+ map = new HashMap<Watcher.Event.EventType, String>();
+
+ map.put(Watcher.Event.EventType.None, ":none");
+ map.put(Watcher.Event.EventType.NodeCreated, ":node-created");
+ map.put(Watcher.Event.EventType.NodeDeleted, ":node-deleted");
+ map.put(Watcher.Event.EventType.NodeDataChanged, ":node-data-changed");
+ map.put(Watcher.Event.EventType.NodeChildrenChanged, ":node-children-changed");
+
+ }
+
+ public static String getTypeName(Watcher.Event.EventType type) {
+ return map.get(type);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java b/storm-core/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java
new file mode 100644
index 0000000..66dc231
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.zookeeper;
+
+import org.apache.zookeeper.Watcher;
+
+import java.util.HashMap;
+
+public class ZkKeeperStates {
+
+ private static HashMap<Watcher.Event.KeeperState, String> map;
+
+ static {
+ map = new HashMap<Watcher.Event.KeeperState, String>();
+
+ map.put(Watcher.Event.KeeperState.AuthFailed, ":auth-failed");
+ map.put(Watcher.Event.KeeperState.SyncConnected, ":connected");
+ map.put(Watcher.Event.KeeperState.Disconnected, ":disconnected");
+ map.put(Watcher.Event.KeeperState.Expired, ":expired");
+ }
+
+ public static String getStateName(Watcher.Event.KeeperState state) {
+ return map.get(state);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/zookeeper/Zookeeper.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/zookeeper/Zookeeper.java b/storm-core/src/jvm/org/apache/storm/zookeeper/Zookeeper.java
new file mode 100644
index 0000000..0580f41
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/zookeeper/Zookeeper.java
@@ -0,0 +1,463 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.zookeeper;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Sets;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.api.CuratorEvent;
+import org.apache.curator.framework.api.CuratorEventType;
+import org.apache.curator.framework.api.CuratorListener;
+import org.apache.curator.framework.recipes.leader.LeaderLatch;
+import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
+import org.apache.curator.framework.recipes.leader.Participant;
+import org.apache.curator.framework.state.ConnectionStateListener;
+import org.apache.storm.Config;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.blobstore.KeyFilter;
+import org.apache.storm.callback.DefaultWatcherCallBack;
+import org.apache.storm.callback.WatcherCallBack;
+import org.apache.storm.cluster.ClusterUtils;
+import org.apache.storm.cluster.IStateStorage;
+import org.apache.storm.cluster.VersionedData;
+import org.apache.storm.nimbus.ILeaderElector;
+import org.apache.storm.nimbus.NimbusInfo;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.Utils;
+import org.apache.storm.utils.ZookeeperAuthInfo;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.server.NIOServerCnxnFactory;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.BindException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
+
+
+public class Zookeeper {
+ private static Logger LOG = LoggerFactory.getLogger(Zookeeper.class);
+
+ // A singleton instance allows us to mock delegated static methods in our
+ // tests by subclassing.
+ private static final Zookeeper INSTANCE = new Zookeeper();
+ private static Zookeeper _instance = INSTANCE;
+
+ /**
+ * Provide an instance of this class for delegates to use. To mock out
+ * delegated methods, provide an instance of a subclass that overrides the
+ * implementation of the delegated method.
+ *
+ * @param u a Zookeeper instance
+ */
+ public static void setInstance(Zookeeper u) {
+ _instance = u;
+ }
+
+ /**
+ * Resets the singleton instance to the default. This is helpful to reset
+ * the class to its original functionality when mocking is no longer
+ * desired.
+ */
+ public static void resetInstance() {
+ _instance = INSTANCE;
+ }
+
+ public CuratorFramework mkClientImpl(Map conf, List<String> servers, Object port, String root) {
+ return mkClientImpl(conf, servers, port, root, new DefaultWatcherCallBack());
+ }
+
+ public CuratorFramework mkClientImpl(Map conf, List<String> servers, Object port, Map authConf) {
+ return mkClientImpl(conf, servers, port, "", new DefaultWatcherCallBack(), authConf);
+ }
+
+ public CuratorFramework mkClientImpl(Map conf, List<String> servers, Object port, String root, Map authConf) {
+ return mkClientImpl(conf, servers, port, root, new DefaultWatcherCallBack(), authConf);
+ }
+
+ public static CuratorFramework mkClient(Map conf, List<String> servers, Object port, String root, final WatcherCallBack watcher, Map authConf) {
+ return _instance.mkClientImpl(conf, servers, port, root, watcher, authConf);
+ }
+
+ public CuratorFramework mkClientImpl(Map conf, List<String> servers, Object port, String root, final WatcherCallBack watcher, Map authConf) {
+ CuratorFramework fk;
+ if (authConf != null) {
+ fk = Utils.newCurator(conf, servers, port, root, new ZookeeperAuthInfo(authConf));
+ } else {
+ fk = Utils.newCurator(conf, servers, port, root);
+ }
+
+ fk.getCuratorListenable().addListener(new CuratorListener() {
+ @Override
+ public void eventReceived(CuratorFramework _fk, CuratorEvent e) throws Exception {
+ if (e.getType().equals(CuratorEventType.WATCHED)) {
+ WatchedEvent event = e.getWatchedEvent();
+ watcher.execute(event.getState(), event.getType(), event.getPath());
+ }
+ }
+ });
+ fk.start();
+ return fk;
+ }
+
+ /**
+ * connect ZK, register Watch/unhandle Watch
+ *
+ * @return
+ */
+ public CuratorFramework mkClientImpl(Map conf, List<String> servers, Object port, String root, final WatcherCallBack watcher) {
+ return mkClientImpl(conf, servers, port, root, watcher, null);
+ }
+
+ public static String createNode(CuratorFramework zk, String path, byte[] data, org.apache.zookeeper.CreateMode mode, List<ACL> acls) {
+ String ret = null;
+ try {
+ String npath = normalizePath(path);
+ ret = zk.create().creatingParentsIfNeeded().withMode(mode).withACL(acls).forPath(npath, data);
+ } catch (Exception e) {
+ throw Utils.wrapInRuntime(e);
+ }
+ return ret;
+ }
+
+ public static String createNode(CuratorFramework zk, String path, byte[] data, List<ACL> acls){
+ return createNode(zk, path, data, org.apache.zookeeper.CreateMode.PERSISTENT, acls);
+ }
+
+ public static boolean existsNode(CuratorFramework zk, String path, boolean watch){
+ Stat stat = null;
+ try {
+ if (watch) {
+ stat = zk.checkExists().watched().forPath(normalizePath(path));
+ } else {
+ stat = zk.checkExists().forPath(normalizePath(path));
+ }
+ } catch (Exception e) {
+ throw Utils.wrapInRuntime(e);
+ }
+ return stat != null;
+ }
+
+ public static void deleteNode(CuratorFramework zk, String path){
+ try {
+ String npath = normalizePath(path);
+ if (existsNode(zk, npath, false)) {
+ zk.delete().deletingChildrenIfNeeded().forPath(normalizePath(path));
+ }
+ } catch (Exception e) {
+ if (Utils.exceptionCauseIsInstanceOf(KeeperException.NodeExistsException.class, e)) {
+ // do nothing
+ LOG.info("delete {} failed.", path, e);
+ } else {
+ throw Utils.wrapInRuntime(e);
+ }
+ }
+ }
+
+ public static void mkdirs(CuratorFramework zk, String path, List<ACL> acls) {
+ _instance.mkdirsImpl(zk, path, acls);
+ }
+
+ public void mkdirsImpl(CuratorFramework zk, String path, List<ACL> acls) {
+ String npath = normalizePath(path);
+ if (npath.equals("/")) {
+ return;
+ }
+ if (existsNode(zk, npath, false)) {
+ return;
+ }
+ byte[] byteArray = new byte[1];
+ byteArray[0] = (byte) 7;
+ try {
+ createNode(zk, npath, byteArray, org.apache.zookeeper.CreateMode.PERSISTENT, acls);
+ } catch (Exception e) {
+ if (Utils.exceptionCauseIsInstanceOf(KeeperException.NodeExistsException.class, e)) {
+ // this can happen when multiple clients doing mkdir at same time
+ }
+ }
+ }
+
+ public static void syncPath(CuratorFramework zk, String path){
+ try {
+ zk.sync().forPath(normalizePath(path));
+ } catch (Exception e) {
+ throw Utils.wrapInRuntime(e);
+ }
+ }
+
+ public static void addListener(CuratorFramework zk, ConnectionStateListener listener) {
+ zk.getConnectionStateListenable().addListener(listener);
+ }
+
+ public static byte[] getData(CuratorFramework zk, String path, boolean watch){
+ try {
+ String npath = normalizePath(path);
+ if (existsNode(zk, npath, watch)) {
+ if (watch) {
+ return zk.getData().watched().forPath(npath);
+ } else {
+ return zk.getData().forPath(npath);
+ }
+ }
+ } catch (Exception e) {
+ if (Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) {
+ // this is fine b/c we still have a watch from the successful exists call
+ } else {
+ throw Utils.wrapInRuntime(e);
+ }
+ }
+ return null;
+ }
+
+ public static Integer getVersion(CuratorFramework zk, String path, boolean watch) throws Exception {
+ String npath = normalizePath(path);
+ Stat stat = null;
+ if (existsNode(zk, npath, watch)) {
+ if (watch) {
+ stat = zk.checkExists().watched().forPath(npath);
+ } else {
+ stat = zk.checkExists().forPath(npath);
+ }
+ }
+ return stat == null ? null : Integer.valueOf(stat.getVersion());
+ }
+
+ public static List<String> getChildren(CuratorFramework zk, String path, boolean watch) {
+ try {
+ String npath = normalizePath(path);
+ if (watch) {
+ return zk.getChildren().watched().forPath(npath);
+ } else {
+ return zk.getChildren().forPath(npath);
+ }
+ } catch (Exception e) {
+ throw Utils.wrapInRuntime(e);
+ }
+ }
+
+ // Deletes the state inside the zookeeper for a key, for which the
+ // contents of the key starts with nimbus host port information
+ public static void deleteNodeBlobstore(CuratorFramework zk, String parentPath, String hostPortInfo){
+ String normalizedPatentPath = normalizePath(parentPath);
+ List<String> childPathList = null;
+ if (existsNode(zk, normalizedPatentPath, false)) {
+ childPathList = getChildren(zk, normalizedPatentPath, false);
+ for (String child : childPathList) {
+ if (child.startsWith(hostPortInfo)) {
+ LOG.debug("deleteNode child {}", child);
+ deleteNode(zk, normalizedPatentPath + "/" + child);
+ }
+ }
+ }
+ }
+
+ public static Stat setData(CuratorFramework zk, String path, byte[] data){
+ try {
+ String npath = normalizePath(path);
+ return zk.setData().forPath(npath, data);
+ } catch (Exception e) {
+ throw Utils.wrapInRuntime(e);
+ }
+ }
+
+ public static boolean exists(CuratorFramework zk, String path, boolean watch){
+ return existsNode(zk, path, watch);
+ }
+
+ public static List mkInprocessZookeeper(String localdir, Integer port) throws Exception {
+ File localfile = new File(localdir);
+ ZooKeeperServer zk = new ZooKeeperServer(localfile, localfile, 2000);
+ NIOServerCnxnFactory factory = null;
+ int report = 2000;
+ int limitPort = 65535;
+ if (port != null) {
+ report = port;
+ limitPort = port;
+ }
+ while (true) {
+ try {
+ factory = new NIOServerCnxnFactory();
+ factory.configure(new InetSocketAddress(report), 0);
+ break;
+ } catch (BindException e) {
+ report++;
+ if (report > limitPort) {
+ throw new RuntimeException("No port is available to launch an inprocess zookeeper");
+ }
+ }
+ }
+ LOG.info("Starting inprocess zookeeper at port {} and dir {}", report, localdir);
+ factory.startup(zk);
+ return Arrays.asList((Object) new Long(report), (Object) factory);
+ }
+
+ public static void shutdownInprocessZookeeper(NIOServerCnxnFactory handle) {
+ handle.shutdown();
+ }
+
+ public static NimbusInfo toNimbusInfo(Participant participant) {
+ String id = participant.getId();
+ if (StringUtils.isBlank(id)) {
+ throw new RuntimeException("No nimbus leader participant host found, have you started your nimbus hosts?");
+ }
+ NimbusInfo nimbusInfo = NimbusInfo.parse(id);
+ nimbusInfo.setLeader(participant.isLeader());
+ return nimbusInfo;
+ }
+
+ // Leader latch listener that will be invoked when we either gain or lose leadership
+ public static LeaderLatchListener leaderLatchListenerImpl(final Map conf, final CuratorFramework zk, final BlobStore blobStore, final LeaderLatch leaderLatch) throws UnknownHostException {
+ final String hostName = InetAddress.getLocalHost().getCanonicalHostName();
+ return new LeaderLatchListener() {
+ @Override
+ public void isLeader() {
+ Set<String> activeTopologyIds = new HashSet<>(Zookeeper.getChildren(zk, conf.get(Config.STORM_ZOOKEEPER_ROOT) + ClusterUtils.STORMS_SUBTREE, false));
+ Set<String> localTopologyIds = blobStore.filterAndListKeys(new KeyFilter<String>() {
+ @Override
+ public String filter(String key) {
+ return ConfigUtils.getIdFromBlobKey(key);
+ }
+ });
+ Sets.SetView<String> diffTopology = Sets.difference(activeTopologyIds, localTopologyIds);
+ LOG.info("active-topology-ids [{}] local-topology-ids [{}] diff-topology [{}]",
+ generateJoinedString(activeTopologyIds), generateJoinedString(localTopologyIds),
+ generateJoinedString(diffTopology));
+
+ if (diffTopology.isEmpty()) {
+ LOG.info("Accepting leadership, all active topology found locally.");
+ } else {
+ LOG.info("code for all active topologies not available locally, giving up leadership.");
+ try {
+ leaderLatch.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void notLeader() {
+ LOG.info("{} lost leadership.", hostName);
+ }
+
+ private String generateJoinedString(Set<String> activeTopologyIds) {
+ return Joiner.on(",").join(activeTopologyIds);
+ }
+ };
+ }
+
+ public static ILeaderElector zkLeaderElector(Map conf, BlobStore blobStore) throws UnknownHostException {
+ return _instance.zkLeaderElectorImpl(conf, blobStore);
+ }
+
+ protected ILeaderElector zkLeaderElectorImpl(Map conf, BlobStore blobStore) throws UnknownHostException {
+ List<String> servers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS);
+ Object port = conf.get(Config.STORM_ZOOKEEPER_PORT);
+ CuratorFramework zk = mkClientImpl(conf, servers, port, "", conf);
+ String leaderLockPath = conf.get(Config.STORM_ZOOKEEPER_ROOT) + "/leader-lock";
+ String id = NimbusInfo.fromConf(conf).toHostPortString();
+ AtomicReference<LeaderLatch> leaderLatchAtomicReference = new AtomicReference<>(new LeaderLatch(zk, leaderLockPath, id));
+ AtomicReference<LeaderLatchListener> leaderLatchListenerAtomicReference =
+ new AtomicReference<>(leaderLatchListenerImpl(conf, zk, blobStore, leaderLatchAtomicReference.get()));
+ return new LeaderElectorImp(conf, servers, zk, leaderLockPath, id, leaderLatchAtomicReference,
+ leaderLatchListenerAtomicReference, blobStore);
+ }
+
+ /**
+ * Get the data along with a version
+ * @param zk the zk instance to use
+ * @param path the path to get it from
+ * @param watch should a watch be enabled
+ * @return null if no data is found, else the data with the version.
+ */
+ public static VersionedData<byte[]> getDataWithVersion(CuratorFramework zk, String path, boolean watch) {
+ VersionedData<byte[]> data = null;
+ try {
+ byte[] bytes = null;
+ Stat stats = new Stat();
+ String npath = normalizePath(path);
+ if (existsNode(zk, npath, watch)) {
+ if (watch) {
+ bytes = zk.getData().storingStatIn(stats).watched().forPath(npath);
+ } else {
+ bytes = zk.getData().storingStatIn(stats).forPath(npath);
+ }
+ if (bytes != null) {
+ int version = stats.getVersion();
+ data = new VersionedData<>(version, bytes);
+ }
+ }
+ } catch (Exception e) {
+ if (Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) {
+ // this is fine b/c we still have a watch from the successful exists call
+ } else {
+ Utils.wrapInRuntime(e);
+ }
+ }
+ return data;
+ }
+
+ public static List<String> tokenizePath(String path) {
+ String[] toks = path.split("/");
+ java.util.ArrayList<String> rtn = new ArrayList<String>();
+ for (String str : toks) {
+ if (!str.isEmpty()) {
+ rtn.add(str);
+ }
+ }
+ return rtn;
+ }
+
+ public static String parentPath(String path) {
+ List<String> toks = Zookeeper.tokenizePath(path);
+ int size = toks.size();
+ if (size > 0) {
+ toks.remove(size - 1);
+ }
+ return Zookeeper.toksToPath(toks);
+ }
+
+ public static String toksToPath(List<String> toks) {
+ StringBuffer buff = new StringBuffer();
+ buff.append("/");
+ int size = toks.size();
+ for (int i = 0; i < size; i++) {
+ buff.append(toks.get(i));
+ if (i < (size - 1)) {
+ buff.append("/");
+ }
+ }
+ return buff.toString();
+ }
+
+ public static String normalizePath(String path) {
+ String rtn = toksToPath(tokenizePath(path));
+ return rtn;
+ }
+}
[20/21] storm git commit: Merge branch 'STORM-2018-1.x' of
https://github.com/revans2/incubator-storm into STORM-2018-1.x-test
Posted by ka...@apache.org.
Merge branch 'STORM-2018-1.x' of https://github.com/revans2/incubator-storm into STORM-2018-1.x-test
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0e0cae4a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0e0cae4a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0e0cae4a
Branch: refs/heads/1.x-branch
Commit: 0e0cae4aff656768e7ca11871856a91db646f71e
Parents: e59447b df33efd
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Oct 28 00:02:45 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Oct 28 00:02:45 2016 +0900
----------------------------------------------------------------------
bin/storm.py | 6 +-
log4j2/cluster.xml | 2 +-
.../cluster_state/zookeeper_state_factory.clj | 5 +-
.../org/apache/storm/command/healthcheck.clj | 88 --
.../org/apache/storm/command/kill_workers.clj | 33 -
.../clj/org/apache/storm/daemon/executor.clj | 4 +-
.../apache/storm/daemon/local_supervisor.clj | 28 +
.../clj/org/apache/storm/daemon/logviewer.clj | 7 +-
.../clj/org/apache/storm/daemon/supervisor.clj | 1272 ------------------
.../src/clj/org/apache/storm/daemon/worker.clj | 1 +
storm-core/src/clj/org/apache/storm/event.clj | 71 -
.../storm/pacemaker/pacemaker_state_factory.clj | 6 +-
storm-core/src/clj/org/apache/storm/testing.clj | 74 +-
.../jvm/org/apache/storm/ProcessSimulator.java | 82 ++
.../src/jvm/org/apache/storm/StormTimer.java | 242 ++++
.../storm/callback/DefaultWatcherCallBack.java | 35 +
.../apache/storm/callback/WatcherCallBack.java | 24 +
.../storm/callback/ZKStateChangedCallback.java | 25 +
.../storm/cluster/ClusterStateFactory.java | 4 +-
.../org/apache/storm/cluster/ClusterUtils.java | 244 ++++
.../org/apache/storm/cluster/ExecutorBeat.java | 44 +
.../org/apache/storm/cluster/IStateStorage.java | 219 +++
.../storm/cluster/IStormClusterState.java | 126 ++
.../storm/cluster/StateStorageFactory.java | 28 +
.../storm/cluster/StormClusterStateImpl.java | 720 ++++++++++
.../org/apache/storm/cluster/VersionedData.java | 36 +
.../apache/storm/cluster/ZKStateStorage.java | 252 ++++
.../org/apache/storm/command/HealthCheck.java | 126 ++
.../org/apache/storm/command/KillWorkers.java | 38 +
.../org/apache/storm/daemon/DaemonCommon.java | 22 +
.../storm/daemon/metrics/MetricsUtils.java | 19 +-
.../storm/daemon/supervisor/AdvancedFSOps.java | 350 +++++
.../storm/daemon/supervisor/BasicContainer.java | 713 ++++++++++
.../supervisor/BasicContainerLauncher.java | 59 +
.../storm/daemon/supervisor/Container.java | 534 ++++++++
.../daemon/supervisor/ContainerLauncher.java | 96 ++
.../supervisor/ContainerRecoveryException.java | 29 +
.../DefaultUncaughtExceptionHandler.java | 31 +
.../supervisor/EventManagerPushCallback.java | 37 +
.../daemon/supervisor/ExitCodeCallback.java | 30 +
.../storm/daemon/supervisor/Killable.java | 50 +
.../storm/daemon/supervisor/LocalContainer.java | 85 ++
.../supervisor/LocalContainerLauncher.java | 60 +
.../daemon/supervisor/ReadClusterState.java | 327 +++++
.../daemon/supervisor/RunAsUserContainer.java | 96 ++
.../supervisor/RunAsUserContainerLauncher.java | 57 +
.../apache/storm/daemon/supervisor/Slot.java | 785 +++++++++++
.../daemon/supervisor/StandaloneSupervisor.java | 83 ++
.../storm/daemon/supervisor/Supervisor.java | 362 +++++
.../daemon/supervisor/SupervisorUtils.java | 321 +++++
.../supervisor/timer/SupervisorHealthCheck.java | 41 +
.../supervisor/timer/SupervisorHeartbeat.java | 87 ++
.../daemon/supervisor/timer/UpdateBlobs.java | 105 ++
.../org/apache/storm/event/EventManager.java | 24 +
.../org/apache/storm/event/EventManagerImp.java | 97 ++
.../apache/storm/localizer/AsyncLocalizer.java | 432 ++++++
.../org/apache/storm/localizer/ILocalizer.java | 70 +
.../localizer/LocalDownloadedResource.java | 146 ++
.../LocalizedResourceRetentionSet.java | 2 +-
.../storm/localizer/LocalizedResourceSet.java | 8 +-
.../org/apache/storm/localizer/Localizer.java | 38 +-
.../storm/metric/StormMetricsRegistry.java | 88 ++
.../org/apache/storm/nimbus/ILeaderElector.java | 6 +-
.../serialization/SerializationFactory.java | 17 +-
.../org/apache/storm/testing/FeederSpout.java | 27 +-
.../staticmocking/MockedSupervisorUtils.java | 31 +
.../apache/storm/trident/util/TridentUtils.java | 2 +-
.../jvm/org/apache/storm/utils/ConfigUtils.java | 536 ++++++++
.../org/apache/storm/utils/InprocMessaging.java | 69 +-
.../jvm/org/apache/storm/utils/LocalState.java | 112 +-
.../org/apache/storm/utils/NimbusClient.java | 19 +-
.../src/jvm/org/apache/storm/utils/Time.java | 51 +-
.../src/jvm/org/apache/storm/utils/Utils.java | 1054 +++++++++++++--
.../storm/zookeeper/LeaderElectorImp.java | 124 ++
.../apache/storm/zookeeper/ZkEventTypes.java | 43 +
.../apache/storm/zookeeper/ZkKeeperStates.java | 41 +
.../org/apache/storm/zookeeper/Zookeeper.java | 463 +++++++
.../src/native/worker-launcher/impl/main.c | 20 +-
.../worker-launcher/impl/worker-launcher.c | 46 +-
.../worker-launcher/impl/worker-launcher.h | 2 +-
.../org/apache/storm/integration_test.clj | 97 +-
.../org/apache/storm/testing4j_test.clj | 11 +-
.../clj/org/apache/storm/logviewer_test.clj | 55 +-
.../test/clj/org/apache/storm/metrics_test.clj | 9 +-
.../test/clj/org/apache/storm/nimbus_test.clj | 10 +-
.../storm/security/auth/drpc_auth_test.clj | 32 +-
.../clj/org/apache/storm/supervisor_test.clj | 751 -----------
.../daemon/supervisor/BasicContainerTest.java | 490 +++++++
.../storm/daemon/supervisor/ContainerTest.java | 265 ++++
.../storm/daemon/supervisor/SlotTest.java | 515 +++++++
.../storm/localizer/AsyncLocalizerTest.java | 190 +++
.../LocalizedResourceRetentionSetTest.java | 10 +-
.../localizer/LocalizedResourceSetTest.java | 12 +-
.../apache/storm/localizer/LocalizerTest.java | 9 +-
94 files changed, 11618 insertions(+), 2627 deletions(-)
----------------------------------------------------------------------
[17/21] storm git commit: Merge branch '1.x-branch' into
STORM-2018-1.x
Posted by ka...@apache.org.
Merge branch '1.x-branch' into STORM-2018-1.x
Conflicts:
storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1d4d414d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1d4d414d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1d4d414d
Branch: refs/heads/1.x-branch
Commit: 1d4d414d9707a2ab4c29b8bba06ea20eef6950a5
Parents: f12b7da a11e470
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Oct 25 14:06:43 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Oct 25 14:06:43 2016 -0500
----------------------------------------------------------------------
.gitignore | 3 +
.travis.yml | 9 +-
CHANGELOG.md | 29 +
bin/storm.cmd | 4 +-
bin/storm.py | 10 +-
conf/defaults.yaml | 2 +-
dev-tools/travis/travis-script.sh | 36 +-
docs/Guaranteeing-message-processing.md | 4 +-
docs/Pacemaker.md | 18 +-
docs/Resource_Aware_Scheduler_overview.md | 253 +++-
docs/STORM-UI-REST-API.md | 111 +-
docs/Trident-API-Overview.md | 17 +
docs/images/Storm-JMS-Example.png | Bin 0 -> 40432 bytes
.../ras_new_strategy_network_cdf_random.png | Bin 0 -> 157859 bytes
...tegy_network_metric_cdf_yahoo_topologies.png | Bin 0 -> 152034 bytes
...rategy_network_metric_improvement_random.png | Bin 0 -> 68976 bytes
.../ras_new_strategy_network_metric_random.png | Bin 0 -> 79004 bytes
...strategy_network_metric_yahoo_topologies.png | Bin 0 -> 75195 bytes
docs/images/ras_new_strategy_runtime_random.png | Bin 0 -> 76534 bytes
docs/images/ras_new_strategy_runtime_yahoo.png | Bin 0 -> 71727 bytes
docs/index.md | 1 +
docs/storm-jms-example.md | 111 ++
docs/storm-jms-spring.md | 25 +
docs/storm-jms.md | 33 +
docs/storm-sql-example.md | 372 +++++
docs/storm-sql-internal.md | 10 +-
docs/storm-sql-reference.md | 1206 +++++++++++++++
docs/storm-sql.md | 54 +-
.../starter/spout/RandomSentenceSpout.java | 39 +-
.../starter/trident/DebugMemoryMapState.java | 73 +
.../starter/trident/TridentKafkaWordCount.java | 58 +-
.../starter/trident/TridentMapExample.java | 11 +-
.../java/org/apache/storm/flux/TCKTest.java | 10 +
.../test/resources/configs/bad_shell_test.yaml | 118 ++
.../src/test/resources/configs/shell_test.yaml | 14 +
.../flux/wrappers/bolts/FluxShellBolt.java | 58 +-
.../flux/wrappers/spouts/FluxShellSpout.java | 55 +-
external/flux/pom.xml | 17 -
external/sql/README.md | 46 +
external/sql/pom.xml | 1 +
external/sql/storm-sql-core/pom.xml | 35 +-
.../storm-sql-core/src/codegen/data/Parser.tdd | 6 +-
.../src/jvm/org/apache/storm/sql/StormSql.java | 5 +
.../jvm/org/apache/storm/sql/StormSqlImpl.java | 46 +-
.../org/apache/storm/sql/StormSqlRunner.java | 55 +-
.../apache/storm/sql/compiler/ExprCompiler.java | 523 -------
.../RexNodeToBlockStatementCompiler.java | 102 ++
.../backends/standalone/PlanCompiler.java | 17 +-
.../backends/standalone/RelNodeCompiler.java | 136 +-
.../compiler/backends/trident/PlanCompiler.java | 9 +-
.../trident/TridentLogicalPlanCompiler.java | 93 +-
.../test/org/apache/storm/sql/TestStormSql.java | 77 +-
.../storm/sql/compiler/TestCompilerUtils.java | 34 +-
.../storm/sql/compiler/TestExprCompiler.java | 93 --
.../storm/sql/compiler/TestExprSemantic.java | 262 +++-
.../backends/standalone/TestPlanCompiler.java | 6 +-
.../standalone/TestRelNodeCompiler.java | 2 +-
.../backends/trident/TestPlanCompiler.java | 211 ++-
.../org/apache/storm/sql/kafka/JsonScheme.java | 58 -
.../apache/storm/sql/kafka/JsonSerializer.java | 56 -
.../sql/kafka/KafkaDataSourcesProvider.java | 61 +-
.../storm/sql/kafka/TestJsonRepresentation.java | 50 -
.../sql/kafka/TestKafkaDataSourcesProvider.java | 26 +-
.../storm-sql-external/storm-sql-redis/pom.xml | 80 +
.../sql/redis/RedisDataSourcesProvider.java | 246 ++++
...apache.storm.sql.runtime.DataSourcesProvider | 16 +
.../sql/redis/TestRedisDataSourcesProvider.java | 162 ++
external/sql/storm-sql-runtime/pom.xml | 13 +-
.../calcite/interpreter/StormContext.java | 31 +
.../sql/runtime/AbstractChannelHandler.java | 32 +-
.../sql/runtime/AbstractValuesProcessor.java | 33 +-
.../storm/sql/runtime/ChannelContext.java | 32 +-
.../storm/sql/runtime/ChannelHandler.java | 32 +-
.../org/apache/storm/sql/runtime/Channels.java | 32 +-
.../apache/storm/sql/runtime/DataSource.java | 32 +-
.../storm/sql/runtime/DataSourcesProvider.java | 33 +-
.../storm/sql/runtime/DataSourcesRegistry.java | 33 +-
.../org/apache/storm/sql/runtime/FieldInfo.java | 4 +-
.../sql/runtime/ISqlTridentDataSource.java | 41 +-
.../sql/runtime/SimpleSqlTridentConsumer.java | 41 +
.../storm/sql/runtime/StormSqlFunctions.java | 32 +-
.../sql/runtime/calcite/StormDataContext.java | 79 +
.../sql/runtime/serde/json/JsonScheme.java | 58 +
.../sql/runtime/serde/json/JsonSerializer.java | 57 +
.../trident/AbstractTridentProcessor.java | 33 +-
.../trident/functions/EvaluationFilter.java | 21 +-
.../trident/functions/EvaluationFunction.java | 22 +-
.../storm/sql/TestJsonRepresentation.java | 52 +
.../test/org/apache/storm/sql/TestUtils.java | 241 ++-
external/storm-cassandra/pom.xml | 15 -
external/storm-eventhubs/pom.xml | 21 +-
.../storm/eventhubs/bolt/EventHubBolt.java | 29 +-
.../eventhubs/spout/BinaryEventDataScheme.java | 66 +
.../storm/eventhubs/spout/EventDataScheme.java | 38 +-
.../storm/eventhubs/spout/EventHubSpout.java | 8 +-
.../eventhubs/spout/EventHubSpoutConfig.java | 119 +-
.../storm/eventhubs/spout/FieldConstants.java | 1 +
.../storm/eventhubs/spout/IEventDataScheme.java | 15 +-
.../eventhubs/spout/StringEventDataScheme.java | 69 +
external/storm-hdfs/pom.xml | 9 -
external/storm-jms/README.markdown | 25 +
external/storm-jms/core/pom.xml | 95 ++
.../apache/storm/jms/JmsMessageProducer.java | 46 +
.../java/org/apache/storm/jms/JmsProvider.java | 48 +
.../org/apache/storm/jms/JmsTupleProducer.java | 58 +
.../java/org/apache/storm/jms/bolt/JmsBolt.java | 217 +++
.../apache/storm/jms/spout/JmsMessageID.java | 58 +
.../org/apache/storm/jms/spout/JmsSpout.java | 382 +++++
.../org/apache/storm/jms/trident/JmsBatch.java | 27 +
.../org/apache/storm/jms/trident/JmsState.java | 129 ++
.../storm/jms/trident/JmsStateFactory.java | 40 +
.../apache/storm/jms/trident/JmsUpdater.java | 38 +
.../storm/jms/trident/TridentJmsSpout.java | 409 +++++
.../apache/storm/jms/spout/JmsSpoutTest.java | 88 ++
.../apache/storm/jms/spout/MockJmsProvider.java | 62 +
.../jms/spout/MockSpoutOutputCollector.java | 55 +
.../storm/jms/spout/MockTupleProducer.java | 47 +
.../core/src/test/resources/jndi.properties | 18 +
external/storm-jms/examples/README.markdown | 12 +
external/storm-jms/examples/pom.xml | 151 ++
.../storm/jms/example/ExampleJmsTopology.java | 131 ++
.../apache/storm/jms/example/GenericBolt.java | 116 ++
.../storm/jms/example/JsonTupleProducer.java | 58 +
.../storm/jms/example/SpringJmsProvider.java | 74 +
.../src/main/resources/jms-activemq.xml | 53 +
.../src/main/resources/log4j.properties | 29 +
external/storm-jms/pom.xml | 70 +
.../jvm/org/apache/storm/kafka/KafkaUtils.java | 9 +-
.../apache/storm/kafka/PartitionManager.java | 2 +-
.../org/apache/storm/kafka/bolt/KafkaBolt.java | 6 +-
.../trident/TransactionalTridentKafkaSpout.java | 8 +-
.../kafka/trident/TridentKafkaEmitter.java | 30 +-
.../storm/redis/bolt/AbstractRedisBolt.java | 5 +
.../common/container/JedisClusterContainer.java | 3 +-
.../JedisCommandsInstanceContainer.java | 10 +-
.../redis/common/container/JedisContainer.java | 3 +-
integration-test/README.md | 59 +
integration-test/config/Vagrantfile | 183 +++
integration-test/config/cluster.xml | 96 ++
integration-test/config/common.sh | 21 +
integration-test/config/etc-hosts | 18 +
integration-test/config/install-storm.sh | 39 +
integration-test/config/install-zookeeper.sh | 20 +
integration-test/config/storm.yaml | 33 +
integration-test/pom.xml | 250 ++++
integration-test/run-it.sh | 83 ++
.../org/apache/storm/ExclamationTopology.java | 93 ++
.../org/apache/storm/debug/DebugHelper.java | 39 +
.../storm/st/topology/TestableTopology.java | 30 +
.../topology/window/SlidingTimeCorrectness.java | 170 +++
.../window/SlidingWindowCorrectness.java | 157 ++
.../window/TumblingTimeCorrectness.java | 167 +++
.../window/TumblingWindowCorrectness.java | 154 ++
.../storm/st/topology/window/data/FromJson.java | 22 +
.../storm/st/topology/window/data/TimeData.java | 110 ++
.../st/topology/window/data/TimeDataWindow.java | 91 ++
.../apache/storm/st/utils/StringDecorator.java | 37 +
.../org/apache/storm/st/utils/TimeUtil.java | 54 +
.../test/java/org/apache/storm/st/DemoTest.java | 85 ++
.../apache/storm/st/helper/AbstractTest.java | 27 +
.../apache/storm/st/meta/TestngListener.java | 97 ++
.../st/tests/window/SlidingWindowTest.java | 187 +++
.../st/tests/window/TumblingWindowTest.java | 99 ++
.../org/apache/storm/st/utils/AssertUtil.java | 71 +
.../org/apache/storm/st/wrapper/LogData.java | 66 +
.../apache/storm/st/wrapper/StormCluster.java | 118 ++
.../org/apache/storm/st/wrapper/TopoWrap.java | 378 +++++
.../src/test/resources/storm-conf/storm.yaml | 33 +
pom.xml | 28 +-
storm-core/pom.xml | 16 +
.../src/clj/org/apache/storm/daemon/nimbus.clj | 118 +-
.../src/clj/org/apache/storm/daemon/worker.clj | 5 +-
.../storm/pacemaker/pacemaker_state_factory.clj | 253 +++-
.../src/clj/org/apache/storm/testing4j.clj | 6 +-
storm-core/src/clj/org/apache/storm/ui/core.clj | 35 +-
storm-core/src/jvm/org/apache/storm/Config.java | 10 +-
.../org/apache/storm/generated/Assignment.java | 244 +--
.../storm/generated/ClusterWorkerHeartbeat.java | 52 +-
.../storm/generated/CommonAggregateStats.java | 166 ++-
.../storm/generated/ComponentPageInfo.java | 386 +++--
.../org/apache/storm/generated/Credentials.java | 44 +-
.../jvm/org/apache/storm/generated/HBNodes.java | 32 +-
.../org/apache/storm/generated/HBRecords.java | 36 +-
.../storm/generated/LSApprovedWorkers.java | 44 +-
.../generated/LSSupervisorAssignments.java | 48 +-
.../apache/storm/generated/LSTopoHistory.java | 64 +-
.../storm/generated/LSTopoHistoryList.java | 36 +-
.../storm/generated/LSWorkerHeartbeat.java | 36 +-
.../apache/storm/generated/ListBlobsResult.java | 32 +-
.../apache/storm/generated/LocalAssignment.java | 36 +-
.../apache/storm/generated/LocalStateData.java | 48 +-
.../org/apache/storm/generated/LogConfig.java | 48 +-
.../jvm/org/apache/storm/generated/Nimbus.java | 36 +-
.../org/apache/storm/generated/NodeInfo.java | 32 +-
.../storm/generated/RebalanceOptions.java | 44 +-
.../storm/generated/SettableBlobMeta.java | 36 +-
.../org/apache/storm/generated/StormBase.java | 92 +-
.../apache/storm/generated/SupervisorInfo.java | 152 +-
.../storm/generated/SupervisorPageInfo.java | 72 +-
.../storm/generated/TopologyHistoryInfo.java | 32 +-
.../storm/generated/TopologyPageInfo.java | 132 +-
.../apache/storm/generated/TopologyStats.java | 220 +--
.../apache/storm/generated/WorkerSummary.java | 44 +-
.../apache/storm/messaging/netty/Client.java | 1 -
.../netty/KerberosSaslClientHandler.java | 6 +-
.../netty/KerberosSaslNettyClient.java | 5 +-
.../jvm/org/apache/storm/nimbus/NimbusInfo.java | 2 +
.../apache/storm/pacemaker/PacemakerClient.java | 72 +-
.../pacemaker/PacemakerConnectionException.java | 24 +
.../pacemaker/codec/ThriftNettyClientCodec.java | 7 +-
.../scheduler/multitenant/IsolatedPool.java | 10 +-
.../storm/scheduler/resource/RAS_Node.java | 12 +
.../storm/scheduler/resource/ResourceUtils.java | 22 +
.../DefaultResourceAwareStrategy.java | 673 +++++----
.../apache/storm/testing/MkClusterParam.java | 14 +-
.../TransactionalSpoutCoordinator.java | 7 +-
.../jvm/org/apache/storm/trident/Stream.java | 40 +
.../OperationAwareFlatMapFunction.java | 24 +
.../operation/OperationAwareMapFunction.java | 24 +
.../storm/trident/operation/builtin/Debug.java | 18 +-
.../operation/impl/FlatMapFunctionExecutor.java | 21 +
.../operation/impl/MapFunctionExecutor.java | 21 +
.../spout/IOpaquePartitionedTridentSpout.java | 11 +-
.../trident/spout/IPartitionedTridentSpout.java | 10 +-
.../OpaquePartitionedTridentSpoutExecutor.java | 62 +-
.../spout/PartitionedTridentSpoutExecutor.java | 24 +-
.../trident/state/CombinerValueUpdater.java | 9 +
.../topology/MasterBatchCoordinator.java | 39 +-
.../state/RotatingTransactionalState.java | 59 +-
.../topology/state/TransactionalState.java | 44 +-
.../org/apache/storm/utils/NimbusClient.java | 4 +-
.../storm/utils/ShellBoltMessageQueue.java | 2 +-
storm-core/src/py/storm/Nimbus.py | 14 +-
storm-core/src/py/storm/ttypes.py | 756 +++++-----
storm-core/src/storm.thrift | 2 +
storm-core/src/ui/public/index.html | 126 +-
storm-core/src/ui/public/js/script.js | 34 +-
.../templates/component-page-template.html | 22 +
.../public/templates/index-page-template.html | 117 +-
.../templates/supervisor-page-template.html | 16 +
.../templates/topology-page-template.html | 102 +-
storm-core/src/ui/public/topology.html | 6 +-
.../org/apache/storm/integration_test.clj | 3 +-
.../storm/pacemaker_state_factory_test.clj | 67 +-
.../resource/TestResourceAwareScheduler.java | 1394 +++++++++---------
.../TestUtilsForResourceAwareScheduler.java | 74 +-
.../eviction/TestDefaultEvictionStrategy.java | 751 ++++++++++
.../TestDefaultResourceAwareStrategy.java | 8 +-
storm-dist/binary/src/main/assembly/binary.xml | 28 +
249 files changed, 15587 insertions(+), 4263 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1d4d414d/bin/storm.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1d4d414d/storm-core/src/clj/org/apache/storm/daemon/worker.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1d4d414d/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
index 077d528,b37b901..2bbeda4
--- a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
+++ b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
@@@ -24,9 -25,10 +25,10 @@@
[util :as util]])
(:import [org.apache.storm.generated
HBExecutionException HBServerMessageType HBMessage
- HBMessageData HBPulse]
+ HBMessageData HBPulse ClusterWorkerHeartbeat]
[org.apache.storm.cluster_state zookeeper_state_factory]
- [org.apache.storm.cluster ClusterState]
+ [org.apache.storm.cluster ClusterState ZKStateStorage]
+ [org.apache.storm.utils Utils]
[org.apache.storm.pacemaker PacemakerClient])
(:gen-class
:implements [org.apache.storm.cluster.ClusterStateFactory]))
@@@ -38,15 -53,108 +53,112 @@@
(defn makeZKState [conf auth-conf acls context]
(.mkState (zookeeper_state_factory.) conf auth-conf acls context))
+(defn -mkStore [this conf auth-conf, acls, context]
+ ;; This is only used by the supervisor (this does not support pacemaker)!!!
+ (ZKStateStorage. conf, auth-conf, acls, context))
+
(def max-retries 10)
+ (defn- delete-worker-hb [path pacemaker-client]
+ (let [response
+ (.send pacemaker-client
+ (HBMessage. HBServerMessageType/DELETE_PATH
+ (HBMessageData/path path)))]
+ (if (= (.get_type response) HBServerMessageType/DELETE_PATH_RESPONSE)
+ :ok
+ (throw (HBExecutionException. "Invalid Response Type")))))
+
+ (defn- get-worker-hb [path pacemaker-client]
+ (let [response (.send pacemaker-client
+ (HBMessage. HBServerMessageType/GET_PULSE
+ (HBMessageData/path path)))]
+ (if (= (.get_type response) HBServerMessageType/GET_PULSE_RESPONSE)
+ (try
+ (.get_details (.get_pulse (.get_data response)))
+ (catch Exception e
+ (throw (HBExecutionException. (.toString e)))))
+ (throw (HBExecutionException. "Invalid Response Type")))))
+
+ (defn- get-worker-hb-children [path pacemaker-client]
+ (let [response (.send pacemaker-client
+ (HBMessage. HBServerMessageType/GET_ALL_NODES_FOR_PATH
+ (HBMessageData/path path)))]
+ (if (= (.get_type response) HBServerMessageType/GET_ALL_NODES_FOR_PATH_RESPONSE)
+ (try
+ (into [] (.get_pulseIds (.get_nodes (.get_data response))))
+ (catch Exception e
+ (throw (HBExecutionException. (.toString e)))))
+ (throw (HBExecutionException. "Invalid Response Type")))))
+
+ (defn shuffle-servers-list [conf]
+ (shuffle (or
+ (conf PACEMAKER-SERVERS)
+ [(conf PACEMAKER-HOST)])))
+
+ (defn shutdown-rotate [servers client-pool]
+ ; Shutdown the client and remove from the pool
+ (when @client-pool
+ (.shutdown (get @client-pool (first @servers)))
+ (swap! client-pool dissoc (first @servers))
+ ; Rotate server list to try another write client
+ (swap! servers (fn [lst] (concat (rest lst) [(first lst)])))))
+
+ (defn get-pacemaker-write-client [conf servers client-pool]
+ ;; Client should be created in case of an exception or first write call
+ ;; Shutdown happens in the retry loop
+ (try
+ (.waitUntilReady
+ (let [client (get @client-pool (first @servers))]
+ (if (nil? client)
+ (do
+ (swap! client-pool merge {(first @servers) (PacemakerClient. conf (first @servers))})
+ (get @client-pool (first @servers)))
+ client)))
+ (catch Exception e
+ (throw e))))
+
+ ;; So we can mock the client for testing
+ (defn makeClientPool [conf client-pool servers]
+ (let [servers @servers
+ current-pool @client-pool]
+ (swap! client-pool merge
+ (into {}
+ (map #(if (get current-pool %)
+ nil
+ [% (PacemakerClient. conf %)])
+ servers)))
+ client-pool))
+
+ ;; Used for mocking in tests
+ (defn is-connection-ready [pacemaker-client]
+ (.isReady pacemaker-client))
+
+ (defn pacemaker-retry-on-exception
+ "Retries specific function on exception based on retries count"
+ [tries task-description f catchfn]
+ (let [res (try {:value (f)}
+ (catch Exception e
+ (do
+ (if catchfn
+ (catchfn e))
+ (if (= 0 tries)
+ (throw e)
+ {:exception e}))))]
+ (if (:exception res)
+ (do
+ (log-error
+ (:exception res)
+ (str "Failed to " task-description ". Will make [" tries "] more attempts."))
+ (recur (dec tries) task-description f catchfn))
+ (do
+ (log-debug (str "Successful " task-description "."))
+ (:value res)))))
+
(defn -mkState [this conf auth-conf acls context]
(let [zk-state (makeZKState conf auth-conf acls context)
- pacemaker-client (makeClient conf)]
+ pacemaker-client-pool (atom {})
+ ;; We shuffle the list to spread clients among servers
+ servers (atom (shuffle-servers-list conf))]
(reify
ClusterState
http://git-wip-us.apache.org/repos/asf/storm/blob/1d4d414d/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/1d4d414d/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
----------------------------------------------------------------------
[21/21] storm git commit: add STORM-2018, 2131, 2117, 2110,
2109 to CHANGELOG
Posted by ka...@apache.org.
add STORM-2018,2131,2117,2110,2109 to CHANGELOG
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/17218ea8
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/17218ea8
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/17218ea8
Branch: refs/heads/1.x-branch
Commit: 17218ea8768f2a51f5da056ee8db8721793dff9d
Parents: 0e0cae4
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Oct 28 00:59:06 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Oct 28 01:04:37 2016 +0900
----------------------------------------------------------------------
CHANGELOG.md | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/17218ea8/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 49418fa..9b76dc9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,9 @@
## 1.1.0
+ * STORM-2109: Under supervisor V2 SUPERVISOR_MEMORY_CAPACITY_MB and SUPERVISOR_CPU_CAPACITY must be Doubles
+ * STORM-2110: in supervisor v2 filter out empty command line args
+ * STORM-2117: Supervisor V2 with local mode extracts resources directory to topology root directory instead of temporary directory
+ * STORM-2131: Add blob command to worker-launcher, make stormdist directory not writeable by topo owner
+ * STORM-2018: Supervisor V2
* STORM-2139: Let ShellBolts and ShellSpouts run with scripts from blobs
* STORM-2072: Add map, flatMap with different outputs (T->V) in Trident
* STORM-2134: improving the current scheduling strategy for RAS
[13/21] storm git commit: Merge branch 'STORM-2109' of
https://github.com/revans2/incubator-storm into STORM-2109
Posted by ka...@apache.org.
Merge branch 'STORM-2109' of https://github.com/revans2/incubator-storm into STORM-2109
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2297757e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2297757e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2297757e
Branch: refs/heads/1.x-branch
Commit: 2297757e100e314bcf7834f403e310b8909d65a8
Parents: 88619b2
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Sun Sep 25 21:26:18 2016 +0900
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Sep 26 08:22:42 2016 -0500
----------------------------------------------------------------------
.../storm/daemon/supervisor/timer/SupervisorHeartbeat.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/2297757e/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
index 34c5682..849c584 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
@@ -72,9 +72,9 @@ public class SupervisorHeartbeat implements Runnable {
private Map<String, Double> mkSupervisorCapacities(Map conf) {
Map<String, Double> ret = new HashMap<String, Double>();
- Double mem = (double) (conf.get(Config.SUPERVISOR_MEMORY_CAPACITY_MB));
+ Double mem = Utils.getDouble(conf.get(Config.SUPERVISOR_MEMORY_CAPACITY_MB), 4096.0);
ret.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, mem);
- Double cpu = (double) (conf.get(Config.SUPERVISOR_CPU_CAPACITY));
+ Double cpu = Utils.getDouble(conf.get(Config.SUPERVISOR_CPU_CAPACITY), 400.0);
ret.put(Config.SUPERVISOR_CPU_CAPACITY, cpu);
return ret;
}
[12/21] storm git commit: Merge branch 'STORM-2110' of
https://github.com/revans2/incubator-storm into STORM-2110
Posted by ka...@apache.org.
Merge branch 'STORM-2110' of https://github.com/revans2/incubator-storm into STORM-2110
Conflicts:
storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/88619b21
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/88619b21
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/88619b21
Branch: refs/heads/1.x-branch
Commit: 88619b2126c9877f7d35b47ba5648cb006637721
Parents: fb42b96
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Sep 23 10:43:33 2016 +0900
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Sep 23 13:00:20 2016 -0500
----------------------------------------------------------------------
.../jvm/org/apache/storm/daemon/supervisor/BasicContainer.java | 6 +++++-
.../org/apache/storm/daemon/supervisor/BasicContainerTest.java | 2 ++
2 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/88619b21/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
index c5b40f2..b1fa226 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -412,7 +412,11 @@ public class BasicContainer extends Container {
String string = substituteChildOptsInternal((String) value, memOnheap);
if (StringUtils.isNotBlank(string)) {
String[] strings = string.split("\\s+");
- rets.addAll(Arrays.asList(strings));
+ for (String s: strings) {
+ if (StringUtils.isNotBlank(s)) {
+ rets.add(s);
+ }
+ }
}
} else if (value instanceof List) {
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/storm/blob/88619b21/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
index 38081f4..3e41125 100644
--- a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
@@ -381,6 +381,7 @@ public class BasicContainerTest {
superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
superConf.put(Config.STORM_LOG4J2_CONF_DIR, log4jdir);
+ superConf.put(Config.WORKER_CHILDOPTS, " -Dtesting=true");
final LocalAssignment la = new LocalAssignment();
la.set_topology_id(topoId);
@@ -432,6 +433,7 @@ public class BasicContainerTest {
"-Dlog4j.configurationFile=" + workerConf,
"-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
"-Dstorm.local.dir=" + stormLocal,
+ "-Dtesting=true",
"-Djava.library.path=JLP",
"-Dstorm.conf.file=",
"-Dstorm.options=",
[04/21] storm git commit: STORM-2018: Supervisor V2
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java
new file mode 100644
index 0000000..d33dc9c
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.testing.staticmocking;
+
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+
+public class MockedSupervisorUtils implements AutoCloseable {
+
+ public MockedSupervisorUtils(SupervisorUtils inst) {
+ SupervisorUtils.setInstance(inst);
+ }
+
+ @Override
+ public void close() throws Exception {
+ SupervisorUtils.resetInstance();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/trident/util/TridentUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/trident/util/TridentUtils.java b/storm-core/src/jvm/org/apache/storm/trident/util/TridentUtils.java
index 8272b3c..f5d317e 100644
--- a/storm-core/src/jvm/org/apache/storm/trident/util/TridentUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/trident/util/TridentUtils.java
@@ -111,7 +111,7 @@ public class TridentUtils {
return Utils.thriftSerialize(t);
}
- public static <T> T thriftDeserialize(Class c, byte[] b) {
+ public static <T> T thriftDeserialize(Class<T> c, byte[] b) {
return Utils.thriftDeserialize(c,b);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java b/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java
new file mode 100644
index 0000000..e2be8a7
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java
@@ -0,0 +1,536 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.utils;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.storm.Config;
+import org.apache.storm.daemon.supervisor.AdvancedFSOps;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.validation.ConfigValidation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class ConfigUtils {
+ private final static Logger LOG = LoggerFactory.getLogger(ConfigUtils.class);
+ public final static String RESOURCES_SUBDIR = "resources";
+ public final static String NIMBUS_DO_NOT_REASSIGN = "NIMBUS-DO-NOT-REASSIGN";
+ public static final String FILE_SEPARATOR = File.separator;
+
+ // A singleton instance allows us to mock delegated static methods in our
+ // tests by subclassing.
+ private static ConfigUtils _instance = new ConfigUtils();
+
+ /**
+ * Provide an instance of this class for delegates to use. To mock out
+ * delegated methods, provide an instance of a subclass that overrides the
+ * implementation of the delegated method.
+ * @param u a Utils instance
+ * @return the previously set instance
+ */
+ public static ConfigUtils setInstance(ConfigUtils u) {
+ ConfigUtils oldInstance = _instance;
+ _instance = u;
+ return oldInstance;
+ }
+
+ public static String getLogDir() {
+ String dir;
+ Map conf;
+ if (System.getProperty("storm.log.dir") != null) {
+ dir = System.getProperty("storm.log.dir");
+ } else if ((conf = readStormConfig()).get("storm.log.dir") != null) {
+ dir = String.valueOf(conf.get("storm.log.dir"));
+ } else if (System.getProperty("storm.home") != null) {
+ dir = System.getProperty("storm.home") + FILE_SEPARATOR + "logs";
+ } else {
+ dir = "logs";
+ }
+ try {
+ return new File(dir).getCanonicalPath();
+ } catch (IOException ex) {
+ throw new IllegalArgumentException("Illegal storm.log.dir in conf: " + dir);
+ }
+ }
+
+ public static String clojureConfigName(String name) {
+ return name.toUpperCase().replace("_", "-");
+ }
+
+ // ALL-CONFIGS is only used by executor.clj once, do we want to do it here? TODO
+ public static List<Object> All_CONFIGS() {
+ List<Object> ret = new ArrayList<Object>();
+ Config config = new Config();
+ Class<?> ConfigClass = config.getClass();
+ Field[] fields = ConfigClass.getFields();
+ for (int i = 0; i < fields.length; i++) {
+ try {
+ Object obj = fields[i].get(null);
+ ret.add(obj);
+ } catch (IllegalArgumentException e) {
+ LOG.error(e.getMessage(), e);
+ } catch (IllegalAccessException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+ return ret;
+ }
+
+ public static String clusterMode(Map conf) {
+ String mode = (String) conf.get(Config.STORM_CLUSTER_MODE);
+ return mode;
+ }
+
+ public static boolean isLocalMode(Map<String, Object> conf) {
+ String mode = (String) conf.get(Config.STORM_CLUSTER_MODE);
+ if (mode != null) {
+ if ("local".equals(mode)) {
+ return true;
+ }
+ if ("distributed".equals(mode)) {
+ return false;
+ }
+ throw new IllegalArgumentException("Illegal cluster mode in conf: " + mode);
+ }
+ return true;
+ }
+
+ public static int samplingRate(Map conf) {
+ double rate = Utils.getDouble(conf.get(Config.TOPOLOGY_STATS_SAMPLE_RATE));
+ if (rate != 0) {
+ return (int) (1 / rate);
+ }
+ throw new IllegalArgumentException("Illegal topology.stats.sample.rate in conf: " + rate);
+ }
+
+ public static Callable<Boolean> evenSampler(final int samplingFreq) {
+ final Random random = new Random();
+
+ return new Callable<Boolean>() {
+ private int curr = -1;
+ private int target = random.nextInt(samplingFreq);
+
+ @Override
+ public Boolean call() throws Exception {
+ curr++;
+ if (curr >= samplingFreq) {
+ curr = 0;
+ target = random.nextInt(samplingFreq);
+ }
+ return (curr == target);
+ }
+ };
+ }
+
+ public static Callable<Boolean> mkStatsSampler(Map conf) {
+ return evenSampler(samplingRate(conf));
+ }
+
+ // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+ public static Map<String, Object> readStormConfig() {
+ return _instance.readStormConfigImpl();
+ }
+
+ public Map<String, Object> readStormConfigImpl() {
+ Map<String, Object> conf = Utils.readStormConfig();
+ ConfigValidation.validateFields(conf);
+ return conf;
+ }
+
+ public static Map readYamlConfig(String name, boolean mustExist) {
+ Map conf = Utils.findAndReadConfigFile(name, mustExist);
+ ConfigValidation.validateFields(conf);
+ return conf;
+ }
+
+ public static Map readYamlConfig(String name) {
+ return readYamlConfig(name, true);
+ }
+
+ public static String absoluteStormLocalDir(Map conf) {
+ String stormHome = System.getProperty("storm.home");
+ String localDir = (String) conf.get(Config.STORM_LOCAL_DIR);
+ if (localDir == null) {
+ return (stormHome + FILE_SEPARATOR + "storm-local");
+ } else {
+ if (new File(localDir).isAbsolute()) {
+ return localDir;
+ } else {
+ return (stormHome + FILE_SEPARATOR + localDir);
+ }
+ }
+ }
+
+ public static String absoluteHealthCheckDir(Map conf) {
+ String stormHome = System.getProperty("storm.home");
+ String healthCheckDir = (String) conf.get(Config.STORM_HEALTH_CHECK_DIR);
+ if (healthCheckDir == null) {
+ return (stormHome + FILE_SEPARATOR + "healthchecks");
+ } else {
+ if (new File(healthCheckDir).isAbsolute()) {
+ return healthCheckDir;
+ } else {
+ return (stormHome + FILE_SEPARATOR + healthCheckDir);
+ }
+ }
+ }
+
+ public static String masterLocalDir(Map conf) throws IOException {
+ String ret = String.valueOf(conf.get(Config.STORM_LOCAL_DIR)) + FILE_SEPARATOR + "nimbus";
+ FileUtils.forceMkdir(new File(ret));
+ return ret;
+ }
+
+ public static String masterStormJarKey(String topologyId) {
+ return (topologyId + "-stormjar.jar");
+ }
+
+ public static String masterStormCodeKey(String topologyId) {
+ return (topologyId + "-stormcode.ser");
+ }
+
+ public static String masterStormConfKey(String topologyId) {
+ return (topologyId + "-stormconf.ser");
+ }
+
+ public static String masterStormDistRoot(Map conf) throws IOException {
+ String ret = stormDistPath(masterLocalDir(conf));
+ FileUtils.forceMkdir(new File(ret));
+ return ret;
+ }
+
+ public static String masterStormDistRoot(Map conf, String stormId) throws IOException {
+ return (masterStormDistRoot(conf) + FILE_SEPARATOR + stormId);
+ }
+
+ public static String stormDistPath(String stormRoot) {
+ String ret = "";
+ // we do this since to concat a null String will actually concat a "null", which is not the expected: ""
+ if (stormRoot != null) {
+ ret = stormRoot;
+ }
+ return ret + FILE_SEPARATOR + "stormdist";
+ }
+
+ public static Map<String, Object> readSupervisorStormConfGivenPath(Map<String, Object> conf, String stormConfPath) throws IOException {
+ Map<String, Object> ret = new HashMap<>(conf);
+ ret.putAll(Utils.fromCompressedJsonConf(FileUtils.readFileToByteArray(new File(stormConfPath))));
+ return ret;
+ }
+
+ public static StormTopology readSupervisorStormCodeGivenPath(String stormCodePath, AdvancedFSOps ops) throws IOException {
+ return Utils.deserialize(ops.slurp(new File(stormCodePath)), StormTopology.class);
+ }
+
+ public static String masterStormJarPath(String stormRoot) {
+ return (stormRoot + FILE_SEPARATOR + "stormjar.jar");
+ }
+
+ public static String masterInbox(Map conf) throws IOException {
+ String ret = masterLocalDir(conf) + FILE_SEPARATOR + "inbox";
+ FileUtils.forceMkdir(new File(ret));
+ return ret;
+ }
+
+ public static String masterInimbusDir(Map conf) throws IOException {
+ return (masterLocalDir(conf) + FILE_SEPARATOR + "inimbus");
+ }
+
+ // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+ public static String supervisorLocalDir(Map conf) throws IOException {
+ return _instance.supervisorLocalDirImpl(conf);
+ }
+
+ public String supervisorLocalDirImpl(Map conf) throws IOException {
+ String ret = absoluteStormLocalDir(conf) + FILE_SEPARATOR + "supervisor";
+ FileUtils.forceMkdir(new File(ret));
+ return ret;
+ }
+
+ public static String supervisorIsupervisorDir(Map conf) throws IOException {
+ return (supervisorLocalDir(conf) + FILE_SEPARATOR + "isupervisor");
+ }
+
+ // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+ public static String supervisorStormDistRoot(Map conf) throws IOException {
+ return _instance.supervisorStormDistRootImpl(conf);
+ }
+
+ public String supervisorStormDistRootImpl(Map conf) throws IOException {
+ return stormDistPath(supervisorLocalDir(conf));
+ }
+
+ // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+ public static String supervisorStormDistRoot(Map conf, String stormId) throws IOException {
+ return _instance.supervisorStormDistRootImpl(conf, stormId);
+ }
+
+ public String supervisorStormDistRootImpl(Map conf, String stormId) throws IOException {
+ return supervisorStormDistRoot(conf) + FILE_SEPARATOR + URLEncoder.encode(stormId, "UTF-8");
+ }
+
+ public static String concatIfNotNull(String dir) {
+ String ret = "";
+ // we do this since to concat a null String will actually concat a "null", which is not the expected: ""
+ if (dir != null) {
+ ret = dir;
+ }
+ return ret;
+ }
+
+ public static String supervisorStormJarPath(String stormRoot) {
+ return (concatIfNotNull(stormRoot) + FILE_SEPARATOR + "stormjar.jar");
+ }
+
+ public static String supervisorStormCodePath(String stormRoot) {
+ return (concatIfNotNull(stormRoot) + FILE_SEPARATOR + "stormcode.ser");
+ }
+
+ public static String supervisorStormConfPath(String stormRoot) {
+ return (concatIfNotNull(stormRoot) + FILE_SEPARATOR + "stormconf.ser");
+ }
+
+ public static String supervisorTmpDir(Map conf) throws IOException {
+ String ret = supervisorLocalDir(conf) + FILE_SEPARATOR + "tmp";
+ FileUtils.forceMkdir(new File(ret));
+ return ret;
+ }
+
+ public static String supervisorStormResourcesPath(String stormRoot) {
+ return (concatIfNotNull(stormRoot) + FILE_SEPARATOR + RESOURCES_SUBDIR);
+ }
+
+ // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+ public static LocalState supervisorState(Map conf) throws IOException {
+ return _instance.supervisorStateImpl(conf);
+ }
+
+ public LocalState supervisorStateImpl(Map conf) throws IOException {
+ return new LocalState((supervisorLocalDir(conf) + FILE_SEPARATOR + "localstate"));
+ }
+
+ // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+ public static LocalState nimbusTopoHistoryState(Map conf) throws IOException {
+ return _instance.nimbusTopoHistoryStateImpl(conf);
+ }
+
+ public LocalState nimbusTopoHistoryStateImpl(Map conf) throws IOException {
+ return new LocalState((masterLocalDir(conf) + FILE_SEPARATOR + "history"));
+ }
+
+ // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+ public static Map<String, Object> readSupervisorStormConf(Map<String, Object> conf, String stormId) throws IOException {
+ return _instance.readSupervisorStormConfImpl(conf, stormId);
+ }
+
+ public Map<String, Object> readSupervisorStormConfImpl(Map<String, Object> conf, String stormId) throws IOException {
+ String stormRoot = supervisorStormDistRoot(conf, stormId);
+ String confPath = supervisorStormConfPath(stormRoot);
+ return readSupervisorStormConfGivenPath(conf, confPath);
+ }
+
+ public static StormTopology readSupervisorTopology(Map conf, String stormId, AdvancedFSOps ops) throws IOException {
+ return _instance.readSupervisorTopologyImpl(conf, stormId, ops);
+ }
+
+ public StormTopology readSupervisorTopologyImpl(Map conf, String stormId, AdvancedFSOps ops) throws IOException {
+ String stormRoot = supervisorStormDistRoot(conf, stormId);
+ String topologyPath = supervisorStormCodePath(stormRoot);
+ return readSupervisorStormCodeGivenPath(topologyPath, ops);
+ }
+
+ public static String workerUserRoot(Map conf) {
+ return (absoluteStormLocalDir(conf) + FILE_SEPARATOR + "workers-users");
+ }
+
+ public static String workerUserFile(Map conf, String workerId) {
+ return (workerUserRoot(conf) + FILE_SEPARATOR + workerId);
+ }
+
+ public static String getIdFromBlobKey(String key) {
+ if (key == null) return null;
+ final String STORM_JAR_SUFFIX = "-stormjar.jar";
+ final String STORM_CODE_SUFFIX = "-stormcode.ser";
+ final String STORM_CONF_SUFFIX = "-stormconf.ser";
+
+ String ret = null;
+ if (key.endsWith(STORM_JAR_SUFFIX)) {
+ ret = key.substring(0, key.length() - STORM_JAR_SUFFIX.length());
+ } else if (key.endsWith(STORM_CODE_SUFFIX)) {
+ ret = key.substring(0, key.length() - STORM_CODE_SUFFIX.length());
+ } else if (key.endsWith(STORM_CONF_SUFFIX)) {
+ ret = key.substring(0, key.length() - STORM_CONF_SUFFIX.length());
+ }
+ return ret;
+ }
+
+ // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+ public static String workerArtifactsRoot(Map conf) {
+ return _instance.workerArtifactsRootImpl(conf);
+ }
+
+ public String workerArtifactsRootImpl(Map conf) {
+ String artifactsDir = (String)conf.get(Config.STORM_WORKERS_ARTIFACTS_DIR);
+ if (artifactsDir == null) {
+ return (getLogDir() + FILE_SEPARATOR + "workers-artifacts");
+ } else {
+ if (new File(artifactsDir).isAbsolute()) {
+ return artifactsDir;
+ } else {
+ return (getLogDir() + FILE_SEPARATOR + artifactsDir);
+ }
+ }
+ }
+
+ public static String workerArtifactsRoot(Map conf, String id) {
+ return (workerArtifactsRoot(conf) + FILE_SEPARATOR + id);
+ }
+
+ public static String workerArtifactsRoot(Map conf, String id, Integer port) {
+ return (workerArtifactsRoot(conf, id) + FILE_SEPARATOR + port);
+ }
+
+ public static String workerArtifactsPidPath(Map conf, String id, Integer port) {
+ return (workerArtifactsRoot(conf, id, port) + FILE_SEPARATOR + "worker.pid");
+ }
+
+ public static File getLogMetaDataFile(String fname) {
+ String[] subStrings = fname.split(FILE_SEPARATOR); // TODO: does this work well on windows?
+ String id = subStrings[0];
+ Integer port = Integer.parseInt(subStrings[1]);
+ return getLogMetaDataFile(Utils.readStormConfig(), id, port);
+ }
+
+ public static File getLogMetaDataFile(Map conf, String id, Integer port) {
+ String fname = workerArtifactsRoot(conf, id, port) + FILE_SEPARATOR + "worker.yaml";
+ return new File(fname);
+ }
+
+ public static File getWorkerDirFromRoot(String logRoot, String id, Integer port) {
+ return new File((logRoot + FILE_SEPARATOR + id + FILE_SEPARATOR + port));
+ }
+
+ // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+ public static String workerRoot(Map conf) {
+ return _instance.workerRootImpl(conf);
+ }
+
+ public String workerRootImpl(Map conf) {
+ return (absoluteStormLocalDir(conf) + FILE_SEPARATOR + "workers");
+ }
+
+ public static String workerRoot(Map conf, String id) {
+ return (workerRoot(conf) + FILE_SEPARATOR + id);
+ }
+
+ public static String workerPidsRoot(Map conf, String id) {
+ return (workerRoot(conf, id) + FILE_SEPARATOR + "pids");
+ }
+
+ public static String workerTmpRoot(Map conf, String id) {
+ return (workerRoot(conf, id) + FILE_SEPARATOR + "tmp");
+ }
+
+
+ public static String workerPidPath(Map conf, String id, String pid) {
+ return (workerPidsRoot(conf, id) + FILE_SEPARATOR + pid);
+ }
+
+ public static String workerPidPath(Map<String, Object> conf, String id, long pid) {
+ return workerPidPath(conf, id, String.valueOf(pid));
+ }
+
+ public static String workerHeartbeatsRoot(Map conf, String id) {
+ return (workerRoot(conf, id) + FILE_SEPARATOR + "heartbeats");
+ }
+
+ public static LocalState workerState(Map conf, String id) throws IOException {
+ return new LocalState(workerHeartbeatsRoot(conf, id));
+ }
+
+ public static Map overrideLoginConfigWithSystemProperty(Map conf) { // note that we delete the return value
+ String loginConfFile = System.getProperty("java.security.auth.login.config");
+ if (loginConfFile != null) {
+ conf.put("java.security.auth.login.config", loginConfFile);
+ }
+ return conf;
+ }
+
+ /* TODO: make sure test these two functions in manual tests */
+ public static List<String> getTopoLogsUsers(Map topologyConf) {
+ List<String> logsUsers = (List<String>)topologyConf.get(Config.LOGS_USERS);
+ List<String> topologyUsers = (List<String>)topologyConf.get(Config.TOPOLOGY_USERS);
+ Set<String> mergedUsers = new HashSet<String>();
+ if (logsUsers != null) {
+ for (String user : logsUsers) {
+ if (user != null) {
+ mergedUsers.add(user);
+ }
+ }
+ }
+ if (topologyUsers != null) {
+ for (String user : topologyUsers) {
+ if (user != null) {
+ mergedUsers.add(user);
+ }
+ }
+ }
+ List<String> ret = new ArrayList<String>(mergedUsers);
+ Collections.sort(ret);
+ return ret;
+ }
+
+ public static List<String> getTopoLogsGroups(Map topologyConf) {
+ List<String> logsGroups = (List<String>)topologyConf.get(Config.LOGS_GROUPS);
+ List<String> topologyGroups = (List<String>)topologyConf.get(Config.TOPOLOGY_GROUPS);
+ Set<String> mergedGroups = new HashSet<String>();
+ if (logsGroups != null) {
+ for (String group : logsGroups) {
+ if (group != null) {
+ mergedGroups.add(group);
+ }
+ }
+ }
+ if (topologyGroups != null) {
+ for (String group : topologyGroups) {
+ if (group != null) {
+ mergedGroups.add(group);
+ }
+ }
+ }
+ List<String> ret = new ArrayList<String>(mergedGroups);
+ Collections.sort(ret);
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/InprocMessaging.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/InprocMessaging.java b/storm-core/src/jvm/org/apache/storm/utils/InprocMessaging.java
index 51250f4..8583e0d 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/InprocMessaging.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/InprocMessaging.java
@@ -19,41 +19,82 @@ package org.apache.storm.utils;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class InprocMessaging {
- private static Map<Integer, LinkedBlockingQueue<Object>> _queues = new HashMap<>();
- private static final Object _lock = new Object();
+ private static Map<Integer, LinkedBlockingQueue<Object>> _queues = new HashMap<Integer, LinkedBlockingQueue<Object>>();
+ private static ConcurrentMap<Integer, AtomicBoolean> _hasReader = new ConcurrentHashMap<>();
private static int port = 1;
+ private static final Logger LOG = LoggerFactory.getLogger(InprocMessaging.class);
- public static int acquireNewPort() {
- int ret;
- synchronized(_lock) {
- ret = port;
- port++;
- }
+ public synchronized static int acquireNewPort() {
+ int ret = port;
+ port++;
return ret;
}
public static void sendMessage(int port, Object msg) {
+ waitForReader(port);
+ getQueue(port).add(msg);
+ }
+
+ public static void sendMessageNoWait(int port, Object msg) {
getQueue(port).add(msg);
}
public static Object takeMessage(int port) throws InterruptedException {
+ readerArrived(port);
return getQueue(port).take();
}
public static Object pollMessage(int port) {
+ readerArrived(port);
return getQueue(port).poll();
- }
+ }
+
+ private static AtomicBoolean getHasReader(int port) {
+ AtomicBoolean ab = _hasReader.get(port);
+ if (ab == null) {
+ _hasReader.putIfAbsent(port, new AtomicBoolean(false));
+ ab = _hasReader.get(port);
+ }
+ return ab;
+ }
- private static LinkedBlockingQueue<Object> getQueue(int port) {
- synchronized(_lock) {
- if(!_queues.containsKey(port)) {
- _queues.put(port, new LinkedBlockingQueue<>());
+ public static void waitForReader(int port) {
+ AtomicBoolean ab = getHasReader(port);
+ long start = Time.currentTimeMillis();
+ while (!ab.get()) {
+ if (Time.isSimulating()) {
+ Time.advanceTime(100);
+ }
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ //Ignored
}
- return _queues.get(port);
+ if (Time.currentTimeMillis() - start > 20000) {
+ LOG.error("DONE WAITING FOR READER AFTER {} ms", Time.currentTimeMillis() - start);
+ break;
+ }
+ }
+ }
+
+ private static void readerArrived(int port) {
+ getHasReader(port).set(true);
+ }
+
+ private synchronized static LinkedBlockingQueue<Object> getQueue(int port) {
+ if(!_queues.containsKey(port)) {
+ _queues.put(port, new LinkedBlockingQueue<Object>());
}
+ return _queues.get(port);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/LocalState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/LocalState.java b/storm-core/src/jvm/org/apache/storm/utils/LocalState.java
index aef1c1c..2f0bb60 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/LocalState.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/LocalState.java
@@ -18,24 +18,28 @@
package org.apache.storm.utils;
import org.apache.commons.io.FileUtils;
+import org.apache.storm.generated.LSApprovedWorkers;
+import org.apache.storm.generated.LSSupervisorAssignments;
+import org.apache.storm.generated.LSSupervisorId;
+import org.apache.storm.generated.LSTopoHistory;
+import org.apache.storm.generated.LSTopoHistoryList;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.LocalStateData;
+import org.apache.storm.generated.ThriftSerializedObject;
+import org.apache.thrift.TBase;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Map;
+import java.util.ArrayList;
import java.util.HashMap;
-import java.io.IOException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.thrift.TBase;
-import org.apache.thrift.TDeserializer;
-import org.apache.thrift.TException;
-import org.apache.thrift.TSerializer;
-
-import org.apache.storm.generated.LocalStateData;
-import org.apache.storm.generated.ThriftSerializedObject;
+import java.util.List;
+import java.util.Map;
/**
* A simple, durable, atomic K/V database. *Very inefficient*, should only be used for occasional reads/writes.
@@ -43,6 +47,11 @@ import org.apache.storm.generated.ThriftSerializedObject;
*/
public class LocalState {
public static final Logger LOG = LoggerFactory.getLogger(LocalState.class);
+ public static final String LS_WORKER_HEARTBEAT = "worker-heartbeat";
+ public static final String LS_ID = "supervisor-id";
+ public static final String LS_LOCAL_ASSIGNMENTS = "local-assignments";
+ public static final String LS_APPROVED_WORKERS = "approved-workers";
+ public static final String LS_TOPO_HISTORY = "topo-hist";
private VersionedStore _vs;
public LocalState(String backingDir) throws IOException {
@@ -157,6 +166,85 @@ public class LocalState {
_vs.cleanup(keepVersions);
}
+ public List<LSTopoHistory> getTopoHistoryList() {
+ LSTopoHistoryList lsTopoHistoryListWrapper = (LSTopoHistoryList) get(LS_TOPO_HISTORY);
+ if (null != lsTopoHistoryListWrapper) {
+ return lsTopoHistoryListWrapper.get_topo_history();
+ }
+ return null;
+ }
+
+ /**
+ * Remove topologies from local state which are older than cutOffAge.
+ * @param cutOffAge
+ */
+ public void filterOldTopologies(long cutOffAge) {
+ LSTopoHistoryList lsTopoHistoryListWrapper = (LSTopoHistoryList) get(LS_TOPO_HISTORY);
+ List<LSTopoHistory> filteredTopoHistoryList = new ArrayList<>();
+ if (null != lsTopoHistoryListWrapper) {
+ for (LSTopoHistory topoHistory : lsTopoHistoryListWrapper.get_topo_history()) {
+ if (topoHistory.get_time_stamp() > cutOffAge) {
+ filteredTopoHistoryList.add(topoHistory);
+ }
+ }
+ }
+ put(LS_TOPO_HISTORY, new LSTopoHistoryList(filteredTopoHistoryList));
+ }
+
+ public void addTopologyHistory(LSTopoHistory lsTopoHistory) {
+ LSTopoHistoryList lsTopoHistoryListWrapper = (LSTopoHistoryList) get(LS_TOPO_HISTORY);
+ List<LSTopoHistory> currentTopoHistoryList = new ArrayList<>();
+ if (null != lsTopoHistoryListWrapper) {
+ currentTopoHistoryList.addAll(lsTopoHistoryListWrapper.get_topo_history());
+ }
+ currentTopoHistoryList.add(lsTopoHistory);
+ put(LS_TOPO_HISTORY, new LSTopoHistoryList(currentTopoHistoryList));
+ }
+
+ public String getSupervisorId() {
+ LSSupervisorId lsSupervisorId = (LSSupervisorId) get(LS_ID);
+ if (null != lsSupervisorId) {
+ return lsSupervisorId.get_supervisor_id();
+ }
+ return null;
+ }
+
+ public void setSupervisorId(String supervisorId) {
+ put(LS_ID, new LSSupervisorId(supervisorId));
+ }
+
+ public Map<String, Integer> getApprovedWorkers() {
+ LSApprovedWorkers lsApprovedWorkers = (LSApprovedWorkers) get(LS_APPROVED_WORKERS);
+ if (null != lsApprovedWorkers) {
+ return lsApprovedWorkers.get_approved_workers();
+ }
+ return null;
+ }
+
+ public void setApprovedWorkers(Map<String, Integer> approvedWorkers) {
+ put(LS_APPROVED_WORKERS, new LSApprovedWorkers(approvedWorkers));
+ }
+
+ public LSWorkerHeartbeat getWorkerHeartBeat() {
+ return (LSWorkerHeartbeat) get(LS_WORKER_HEARTBEAT);
+ }
+
+ public void setWorkerHeartBeat(LSWorkerHeartbeat workerHeartBeat) {
+ put(LS_WORKER_HEARTBEAT, workerHeartBeat, false);
+ }
+
+ public Map<Integer, LocalAssignment> getLocalAssignmentsMap() {
+ LSSupervisorAssignments assignments = (LSSupervisorAssignments) get(LS_LOCAL_ASSIGNMENTS);
+ if (null != assignments) {
+ return assignments.get_assignments();
+ }
+ return null;
+ }
+
+ public void setLocalAssignmentsMap(Map<Integer, LocalAssignment> localAssignmentMap) {
+ put(LS_LOCAL_ASSIGNMENTS, new LSSupervisorAssignments(localAssignmentMap));
+ }
+
private void persistInternal(Map<String, ThriftSerializedObject> serialized, TSerializer ser, boolean cleanup) {
try {
if (ser == null) {
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
index 456b459..0e5cef9 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
@@ -17,11 +17,11 @@
*/
package org.apache.storm.utils;
-
import org.apache.storm.Config;
import org.apache.storm.generated.ClusterSummary;
import org.apache.storm.generated.Nimbus;
import org.apache.storm.generated.NimbusSummary;
+import org.apache.storm.security.auth.ReqContext;
import org.apache.storm.security.auth.ThriftClient;
import org.apache.storm.security.auth.ThriftConnectionType;
import com.google.common.collect.Lists;
@@ -29,6 +29,7 @@ import org.apache.thrift.transport.TTransportException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.security.Principal;
import java.util.List;
import java.util.Map;
@@ -36,6 +37,22 @@ public class NimbusClient extends ThriftClient {
private Nimbus.Client _client;
private static final Logger LOG = LoggerFactory.getLogger(NimbusClient.class);
+ public interface WithNimbus {
+ public void run(Nimbus.Client client) throws Exception;
+ }
+
+ public static void withConfiguredClient(WithNimbus cb) throws Exception {
+ withConfiguredClient(cb, ConfigUtils.readStormConfig());
+ }
+
+ public static void withConfiguredClient(WithNimbus cb, Map conf) throws Exception {
+ ReqContext context = ReqContext.context();
+ Principal principal = context.principal();
+ String user = principal == null ? null : principal.getName();
+ try (NimbusClient client = getConfiguredClientAs(conf, user);) {
+ cb.run(client.getClient());
+ }
+ }
public static NimbusClient getConfiguredClient(Map conf) {
return getConfiguredClientAs(conf, null);
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/Time.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/Time.java b/storm-core/src/jvm/org/apache/storm/utils/Time.java
index 1792252..e501b6c 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/Time.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/Time.java
@@ -29,6 +29,7 @@ public class Time {
public static final Logger LOG = LoggerFactory.getLogger(Time.class);
private static AtomicBoolean simulating = new AtomicBoolean(false);
+ private static AtomicLong autoAdvanceOnSleep = new AtomicLong(0);
//TODO: should probably use weak references here or something
private static volatile Map<Thread, AtomicLong> threadSleepTimes;
private static final Object sleepTimesLock = new Object();
@@ -43,10 +44,18 @@ public class Time {
}
}
+ public static void startSimulatingAutoAdvanceOnSleep(long ms) {
+ synchronized(sleepTimesLock) {
+ startSimulating();
+ autoAdvanceOnSleep.set(ms);
+ }
+ }
+
public static void stopSimulating() {
synchronized(sleepTimesLock) {
- simulating.set(false);
- threadSleepTimes = null;
+ simulating.set(false);
+ autoAdvanceOnSleep.set(0);
+ threadSleepTimes = null;
}
}
@@ -71,6 +80,10 @@ public class Time {
throw new InterruptedException();
}
}
+ long autoAdvance = autoAdvanceOnSleep.get();
+ if (autoAdvance > 0) {
+ advanceTime(autoAdvance);
+ }
Thread.sleep(10);
}
} finally {
@@ -86,10 +99,16 @@ public class Time {
Thread.sleep(sleepTime);
}
}
-
+
public static void sleep(long ms) throws InterruptedException {
sleepUntil(currentTimeMillis()+ms);
}
+
+ public static void sleepSecs (long secs) throws InterruptedException {
+ if (secs > 0) {
+ sleep(secs * 1000);
+ }
+ }
public static long currentTimeMillis() {
if(simulating.get()) {
@@ -98,14 +117,32 @@ public class Time {
return System.currentTimeMillis();
}
}
-
+
+ public static long secsToMillis (int secs) {
+ return 1000*(long) secs;
+ }
+
+ public static long secsToMillisLong(double secs) {
+ return (long) (1000 * secs);
+ }
+
public static int currentTimeSecs() {
return (int) (currentTimeMillis() / 1000);
}
+
+ public static int deltaSecs(int timeInSeconds) {
+ return Time.currentTimeSecs() - timeInSeconds;
+ }
+
+ public static long deltaMs(long timeInMilliseconds) {
+ return Time.currentTimeMillis() - timeInMilliseconds;
+ }
public static void advanceTime(long ms) {
- if(!simulating.get()) throw new IllegalStateException("Cannot simulate time unless in simulation mode");
- simulatedCurrTimeMs.set(simulatedCurrTimeMs.get() + ms);
+ if (!simulating.get()) throw new IllegalStateException("Cannot simulate time unless in simulation mode");
+ if (ms < 0) throw new IllegalArgumentException("advanceTime only accepts positive time as an argument");
+ long newTime = simulatedCurrTimeMs.addAndGet(ms);
+ LOG.warn("Advanced simulated time to {}", newTime);
}
public static boolean isThreadWaiting(Thread t) {
@@ -115,5 +152,5 @@ public class Time {
time = threadSleepTimes.get(t);
}
return !t.isAlive() || time!=null && currentTimeMillis() < time.longValue();
- }
+ }
}
[16/21] storm git commit: Another java 7 fix
Posted by ka...@apache.org.
Another java 7 fix
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f12b7dae
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f12b7dae
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f12b7dae
Branch: refs/heads/1.x-branch
Commit: f12b7daeb759002f8a578c32c3fd70c1315644c7
Parents: 905c443
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Sep 26 09:54:03 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Sep 26 09:54:03 2016 -0500
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f12b7dae/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
index 977cb2b..daa1d00 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -529,7 +529,7 @@ public class BasicContainer extends Container {
//Only go off of the topology id for now.
DependencyLocations dl = _cache.get(topologyId);
if (dl == null) {
- _cache.putIfAbsent(topologyId, new DependencyLocations(conf, topologyId, ops, stormRoot));
+ _cache.put(topologyId, new DependencyLocations(conf, topologyId, ops, stormRoot));
dl = _cache.get(topologyId);
}
return dl;
[05/21] storm git commit: STORM-2018: Supervisor V2
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
new file mode 100644
index 0000000..5e89eea
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
@@ -0,0 +1,357 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.storm.Config;
+import org.apache.storm.StormTimer;
+import org.apache.storm.cluster.ClusterStateContext;
+import org.apache.storm.cluster.ClusterUtils;
+import org.apache.storm.cluster.DaemonType;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.daemon.DaemonCommon;
+import org.apache.storm.daemon.supervisor.timer.SupervisorHealthCheck;
+import org.apache.storm.daemon.supervisor.timer.SupervisorHeartbeat;
+import org.apache.storm.daemon.supervisor.timer.UpdateBlobs;
+import org.apache.storm.event.EventManager;
+import org.apache.storm.event.EventManagerImp;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.localizer.AsyncLocalizer;
+import org.apache.storm.localizer.ILocalizer;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.messaging.IContext;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.apache.storm.utils.VersionInfo;
+import org.apache.zookeeper.data.ACL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Supervisor implements DaemonCommon, AutoCloseable {
+ private static final Logger LOG = LoggerFactory.getLogger(Supervisor.class);
+ private final Map<String, Object> conf;
+ private final IContext sharedContext;
+ private volatile boolean active;
+ private final ISupervisor iSupervisor;
+ private final Utils.UptimeComputer upTime;
+ private final String stormVersion;
+ private final IStormClusterState stormClusterState;
+ private final LocalState localState;
+ private final String supervisorId;
+ private final String assignmentId;
+ private final String hostName;
+ // used for reporting used ports when heartbeating
+ private final AtomicReference<Map<Long, LocalAssignment>> currAssignment;
+ private final StormTimer heartbeatTimer;
+ private final StormTimer eventTimer;
+ private final StormTimer blobUpdateTimer;
+ private final Localizer localizer;
+ private final ILocalizer asyncLocalizer;
+ private EventManager eventManager;
+ private ReadClusterState readState;
+
+ private Supervisor(ISupervisor iSupervisor) throws IOException {
+ this(Utils.readStormConfig(), null, iSupervisor);
+ }
+
+ public Supervisor(Map<String, Object> conf, IContext sharedContext, ISupervisor iSupervisor) throws IOException {
+ this.conf = conf;
+ this.iSupervisor = iSupervisor;
+ this.active = true;
+ this.upTime = Utils.makeUptimeComputer();
+ this.stormVersion = VersionInfo.getVersion();
+ this.sharedContext = sharedContext;
+
+ iSupervisor.prepare(conf, ConfigUtils.supervisorIsupervisorDir(conf));
+
+ List<ACL> acls = null;
+ if (Utils.isZkAuthenticationConfiguredStormServer(conf)) {
+ acls = SupervisorUtils.supervisorZkAcls();
+ }
+
+ try {
+ this.stormClusterState = ClusterUtils.mkStormClusterState(conf, acls, new ClusterStateContext(DaemonType.SUPERVISOR));
+ } catch (Exception e) {
+ LOG.error("supervisor can't create stormClusterState");
+ throw Utils.wrapInRuntime(e);
+ }
+
+ try {
+ this.localState = ConfigUtils.supervisorState(conf);
+ this.localizer = Utils.createLocalizer(conf, ConfigUtils.supervisorLocalDir(conf));
+ this.asyncLocalizer = new AsyncLocalizer(conf, this.localizer);
+ } catch (IOException e) {
+ throw Utils.wrapInRuntime(e);
+ }
+ this.supervisorId = iSupervisor.getSupervisorId();
+ this.assignmentId = iSupervisor.getAssignmentId();
+
+ try {
+ this.hostName = Utils.hostname(conf);
+ } catch (UnknownHostException e) {
+ throw Utils.wrapInRuntime(e);
+ }
+
+ this.currAssignment = new AtomicReference<Map<Long, LocalAssignment>>(new HashMap<Long,LocalAssignment>());
+
+ this.heartbeatTimer = new StormTimer(null, new DefaultUncaughtExceptionHandler());
+
+ this.eventTimer = new StormTimer(null, new DefaultUncaughtExceptionHandler());
+
+ this.blobUpdateTimer = new StormTimer("blob-update-timer", new DefaultUncaughtExceptionHandler());
+ }
+
+ public String getId() {
+ return supervisorId;
+ }
+
+ IContext getSharedContext() {
+ return sharedContext;
+ }
+
+ public Map<String, Object> getConf() {
+ return conf;
+ }
+
+ public ISupervisor getiSupervisor() {
+ return iSupervisor;
+ }
+
+ public Utils.UptimeComputer getUpTime() {
+ return upTime;
+ }
+
+ public String getStormVersion() {
+ return stormVersion;
+ }
+
+ public IStormClusterState getStormClusterState() {
+ return stormClusterState;
+ }
+
+ LocalState getLocalState() {
+ return localState;
+ }
+
+ public String getAssignmentId() {
+ return assignmentId;
+ }
+
+ public String getHostName() {
+ return hostName;
+ }
+
+ public AtomicReference<Map<Long, LocalAssignment>> getCurrAssignment() {
+ return currAssignment;
+ }
+
+ public Localizer getLocalizer() {
+ return localizer;
+ }
+
+ ILocalizer getAsyncLocalizer() {
+ return asyncLocalizer;
+ }
+
+ EventManager getEventManger() {
+ return eventManager;
+ }
+
+ /**
+ * Launch the supervisor
+ */
+ public void launch() throws Exception {
+ LOG.info("Starting Supervisor with conf {}", conf);
+ String path = ConfigUtils.supervisorTmpDir(conf);
+ FileUtils.cleanDirectory(new File(path));
+
+ Localizer localizer = getLocalizer();
+
+ SupervisorHeartbeat hb = new SupervisorHeartbeat(conf, this);
+ hb.run();
+ // should synchronize supervisor so it doesn't launch anything after being down (optimization)
+ Integer heartbeatFrequency = Utils.getInt(conf.get(Config.SUPERVISOR_HEARTBEAT_FREQUENCY_SECS));
+ heartbeatTimer.scheduleRecurring(0, heartbeatFrequency, hb);
+
+ this.eventManager = new EventManagerImp(false);
+ this.readState = new ReadClusterState(this);
+
+ Set<String> downloadedTopoIds = SupervisorUtils.readDownloadedTopologyIds(conf);
+ for (String topoId : downloadedTopoIds) {
+ SupervisorUtils.addBlobReferences(localizer, topoId, conf);
+ }
+ // do this after adding the references so we don't try to clean things being used
+ localizer.startCleaner();
+
+ UpdateBlobs updateBlobsThread = new UpdateBlobs(this);
+
+ if ((Boolean) conf.get(Config.SUPERVISOR_ENABLE)) {
+ // This isn't strictly necessary, but it doesn't hurt and ensures that the machine stays up
+ // to date even if callbacks don't all work exactly right
+ eventTimer.scheduleRecurring(0, 10, new EventManagerPushCallback(readState, eventManager));
+
+ // Blob update thread. Starts with 30 seconds delay, every 30 seconds
+ blobUpdateTimer.scheduleRecurring(30, 30, new EventManagerPushCallback(updateBlobsThread, eventManager));
+
+ // supervisor health check
+ eventTimer.scheduleRecurring(300, 300, new SupervisorHealthCheck(this));
+ }
+ LOG.info("Starting supervisor with id {} at host {}.", getId(), getHostName());
+ }
+
+ /**
+ * start distribute supervisor
+ */
+ private void launchDaemon() {
+ LOG.info("Starting supervisor for storm version '{}'.", VersionInfo.getVersion());
+ try {
+ Map<String, Object> conf = getConf();
+ if (ConfigUtils.isLocalMode(conf)) {
+ throw new IllegalArgumentException("Cannot start server in local mode!");
+ }
+ launch();
+ Utils.addShutdownHookWithForceKillIn1Sec(() -> {this.close();});
+ registerWorkerNumGauge("supervisor:num-slots-used-gauge", conf);
+ StormMetricsRegistry.startMetricsReporters(conf);
+ } catch (Exception e) {
+ LOG.error("Failed to start supervisor\n", e);
+ System.exit(1);
+ }
+ }
+
+ private void registerWorkerNumGauge(String name, final Map<String, Object> conf) {
+ StormMetricsRegistry.registerGauge(name, new Callable<Integer>() {
+ @Override
+ public Integer call() throws Exception {
+ Collection<String> pids = SupervisorUtils.supervisorWorkerIds(conf);
+ return pids.size();
+ }
+ });
+ }
+
+ @Override
+ public void close() {
+ try {
+ LOG.info("Shutting down supervisor {}", getId());
+ this.active = false;
+ heartbeatTimer.close();
+ eventTimer.close();
+ blobUpdateTimer.close();
+ if (eventManager != null) {
+ eventManager.close();
+ }
+ if (readState != null) {
+ readState.close();
+ }
+ getStormClusterState().disconnect();
+ } catch (Exception e) {
+ LOG.error("Error Shutting down", e);
+ }
+ }
+
+ void killWorkers(Collection<String> workerIds, ContainerLauncher launcher) throws InterruptedException, IOException {
+ HashSet<Killable> containers = new HashSet<>();
+ for (String workerId : workerIds) {
+ try {
+ Killable k = launcher.recoverContainer(workerId, localState);
+ if (!k.areAllProcessesDead()) {
+ k.kill();
+ containers.add(k);
+ } else {
+ k.cleanUp();
+ }
+ } catch (Exception e) {
+ LOG.error("Error trying to kill {}", workerId, e);
+ }
+ }
+ int shutdownSleepSecs = Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS), 1);
+ if (!containers.isEmpty()) {
+ Time.sleepSecs(shutdownSleepSecs);
+ }
+ for (Killable k: containers) {
+ try {
+ k.forceKill();
+ long start = Time.currentTimeMillis();
+ while(!k.areAllProcessesDead()) {
+ if ((Time.currentTimeMillis() - start) > 10_000) {
+ throw new RuntimeException("Giving up on killing " + k
+ + " after " + (Time.currentTimeMillis() - start) + " ms");
+ }
+ Time.sleep(100);
+ k.forceKill();
+ }
+ k.cleanUp();
+ } catch (Exception e) {
+ LOG.error("Error trying to clean up {}", k, e);
+ }
+ }
+ }
+
+ public void shutdownAllWorkers() {
+ if (readState != null) {
+ readState.shutdownAllWorkers();
+ } else {
+ try {
+ ContainerLauncher launcher = ContainerLauncher.make(getConf(), getId(), getSharedContext());
+ killWorkers(SupervisorUtils.supervisorWorkerIds(conf), launcher);
+ } catch (Exception e) {
+ throw Utils.wrapInRuntime(e);
+ }
+ }
+ }
+
+ @Override
+ public boolean isWaiting() {
+ if (!active) {
+ return true;
+ }
+
+ if (heartbeatTimer.isTimerWaiting() && eventTimer.isTimerWaiting() && eventManager.waiting()) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * supervisor daemon enter entrance
+ *
+ * @param args
+ */
+ public static void main(String[] args) throws Exception {
+ Utils.setupDefaultUncaughtExceptionHandler();
+ @SuppressWarnings("resource")
+ Supervisor instance = new Supervisor(new StandaloneSupervisor());
+ instance.launchDaemon();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
new file mode 100644
index 0000000..0784631
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
@@ -0,0 +1,311 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.Config;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.localizer.LocalResource;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class SupervisorUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SupervisorUtils.class);
+
+ private static final SupervisorUtils INSTANCE = new SupervisorUtils();
+ private static SupervisorUtils _instance = INSTANCE;
+ public static void setInstance(SupervisorUtils u) {
+ _instance = u;
+ }
+ public static void resetInstance() {
+ _instance = INSTANCE;
+ }
+
+ static Process processLauncher(Map<String, Object> conf, String user, List<String> commandPrefix, List<String> args, Map<String, String> environment, final String logPreFix,
+ final ExitCodeCallback exitCodeCallback, File dir) throws IOException {
+ if (StringUtils.isBlank(user)) {
+ throw new IllegalArgumentException("User cannot be blank when calling processLauncher.");
+ }
+ String wlinitial = (String) (conf.get(Config.SUPERVISOR_WORKER_LAUNCHER));
+ String stormHome = ConfigUtils.concatIfNotNull(System.getProperty("storm.home"));
+ String wl;
+ if (StringUtils.isNotBlank(wlinitial)) {
+ wl = wlinitial;
+ } else {
+ wl = stormHome + "/bin/worker-launcher";
+ }
+ List<String> commands = new ArrayList<>();
+ if (commandPrefix != null){
+ commands.addAll(commandPrefix);
+ }
+ commands.add(wl);
+ commands.add(user);
+ commands.addAll(args);
+ LOG.info("Running as user: {} command: {}", user, commands);
+ return SupervisorUtils.launchProcess(commands, environment, logPreFix, exitCodeCallback, dir);
+ }
+
+ public static int processLauncherAndWait(Map<String, Object> conf, String user, List<String> args, final Map<String, String> environment, final String logPreFix)
+ throws IOException {
+ int ret = 0;
+ Process process = processLauncher(conf, user, null, args, environment, logPreFix, null, null);
+ if (StringUtils.isNotBlank(logPreFix))
+ Utils.readAndLogStream(logPreFix, process.getInputStream());
+ try {
+ process.waitFor();
+ } catch (InterruptedException e) {
+ LOG.info("{} interrupted.", logPreFix);
+ }
+ ret = process.exitValue();
+ return ret;
+ }
+
+ public static void setupStormCodeDir(Map<String, Object> conf, Map<String, Object> stormConf, String dir) throws IOException {
+ if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
+ String logPrefix = "setup conf for " + dir;
+ List<String> commands = new ArrayList<>();
+ commands.add("code-dir");
+ commands.add(dir);
+ processLauncherAndWait(conf, (String) (stormConf.get(Config.TOPOLOGY_SUBMITTER_USER)), commands, null, logPrefix);
+ }
+ }
+
+ public static void rmrAsUser(Map<String, Object> conf, String id, String path) throws IOException {
+ String user = Utils.getFileOwner(path);
+ String logPreFix = "rmr " + id;
+ List<String> commands = new ArrayList<>();
+ commands.add("rmr");
+ commands.add(path);
+ SupervisorUtils.processLauncherAndWait(conf, user, commands, null, logPreFix);
+ if (Utils.checkFileExists(path)) {
+ throw new RuntimeException(path + " was not deleted.");
+ }
+ }
+
+ /**
+ * Given the blob information returns the value of the uncompress field, handling it either being a string or a boolean value, or if it's not specified then
+ * returns false
+ *
+ * @param blobInfo
+ * @return
+ */
+ public static Boolean shouldUncompressBlob(Map<String, Object> blobInfo) {
+ return Utils.getBoolean(blobInfo.get("uncompress"), false);
+ }
+
+ /**
+ * Returns a list of LocalResources based on the blobstore-map passed in
+ *
+ * @param blobstoreMap
+ * @return
+ */
+ public static List<LocalResource> blobstoreMapToLocalresources(Map<String, Map<String, Object>> blobstoreMap) {
+ List<LocalResource> localResourceList = new ArrayList<>();
+ if (blobstoreMap != null) {
+ for (Map.Entry<String, Map<String, Object>> map : blobstoreMap.entrySet()) {
+ LocalResource localResource = new LocalResource(map.getKey(), shouldUncompressBlob(map.getValue()));
+ localResourceList.add(localResource);
+ }
+ }
+ return localResourceList;
+ }
+
+ /**
+ * For each of the downloaded topologies, adds references to the blobs that the topologies are using. This is used to reconstruct the cache on restart.
+ *
+ * @param localizer
+ * @param stormId
+ * @param conf
+ */
+ static void addBlobReferences(Localizer localizer, String stormId, Map<String, Object> conf) throws IOException {
+ Map<String, Object> stormConf = ConfigUtils.readSupervisorStormConf(conf, stormId);
+ Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) stormConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
+ String user = (String) stormConf.get(Config.TOPOLOGY_SUBMITTER_USER);
+ String topoName = (String) stormConf.get(Config.TOPOLOGY_NAME);
+ List<LocalResource> localresources = SupervisorUtils.blobstoreMapToLocalresources(blobstoreMap);
+ if (blobstoreMap != null) {
+ localizer.addReferences(localresources, user, topoName);
+ }
+ }
+
+ public static Set<String> readDownloadedTopologyIds(Map<String, Object> conf) throws IOException {
+ Set<String> stormIds = new HashSet<>();
+ String path = ConfigUtils.supervisorStormDistRoot(conf);
+ Collection<String> rets = Utils.readDirContents(path);
+ for (String ret : rets) {
+ stormIds.add(URLDecoder.decode(ret));
+ }
+ return stormIds;
+ }
+
+ public static Collection<String> supervisorWorkerIds(Map<String, Object> conf) {
+ String workerRoot = ConfigUtils.workerRoot(conf);
+ return Utils.readDirContents(workerRoot);
+ }
+
+ static boolean doRequiredTopoFilesExist(Map<String, Object> conf, String stormId) throws IOException {
+ String stormroot = ConfigUtils.supervisorStormDistRoot(conf, stormId);
+ String stormjarpath = ConfigUtils.supervisorStormJarPath(stormroot);
+ String stormcodepath = ConfigUtils.supervisorStormCodePath(stormroot);
+ String stormconfpath = ConfigUtils.supervisorStormConfPath(stormroot);
+ if (!Utils.checkFileExists(stormroot))
+ return false;
+ if (!Utils.checkFileExists(stormcodepath))
+ return false;
+ if (!Utils.checkFileExists(stormconfpath))
+ return false;
+ if (ConfigUtils.isLocalMode(conf) || Utils.checkFileExists(stormjarpath))
+ return true;
+ return false;
+ }
+
+ /**
+ * map from worker id to heartbeat
+ *
+ * @param conf
+ * @return
+ * @throws Exception
+ */
+ public static Map<String, LSWorkerHeartbeat> readWorkerHeartbeats(Map<String, Object> conf) throws Exception {
+ return _instance.readWorkerHeartbeatsImpl(conf);
+ }
+
+ public Map<String, LSWorkerHeartbeat> readWorkerHeartbeatsImpl(Map<String, Object> conf) throws Exception {
+ Map<String, LSWorkerHeartbeat> workerHeartbeats = new HashMap<>();
+
+ Collection<String> workerIds = SupervisorUtils.supervisorWorkerIds(conf);
+
+ for (String workerId : workerIds) {
+ LSWorkerHeartbeat whb = readWorkerHeartbeat(conf, workerId);
+ // ATTENTION: whb can be null
+ workerHeartbeats.put(workerId, whb);
+ }
+ return workerHeartbeats;
+ }
+
+
+ /**
+ * get worker heartbeat by workerId
+ *
+ * @param conf
+ * @param workerId
+ * @return
+ * @throws IOException
+ */
+ private static LSWorkerHeartbeat readWorkerHeartbeat(Map<String, Object> conf, String workerId) {
+ return _instance.readWorkerHeartbeatImpl(conf, workerId);
+ }
+
+ protected LSWorkerHeartbeat readWorkerHeartbeatImpl(Map<String, Object> conf, String workerId) {
+ try {
+ LocalState localState = ConfigUtils.workerState(conf, workerId);
+ return localState.getWorkerHeartBeat();
+ } catch (Exception e) {
+ LOG.warn("Failed to read local heartbeat for workerId : {},Ignoring exception.", workerId, e);
+ return null;
+ }
+ }
+
+ public static boolean isWorkerHbTimedOut(int now, LSWorkerHeartbeat whb, Map<String, Object> conf) {
+ return _instance.isWorkerHbTimedOutImpl(now, whb, conf);
+ }
+
+ private boolean isWorkerHbTimedOutImpl(int now, LSWorkerHeartbeat whb, Map<String, Object> conf) {
+ return (now - whb.get_time_secs()) > Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS));
+ }
+
+ /**
+ * Launch a new process as per {@link java.lang.ProcessBuilder} with a given
+ * callback.
+ * @param command the command to be executed in the new process
+ * @param environment the environment to be applied to the process. Can be
+ * null.
+ * @param logPrefix a prefix for log entries from the output of the process.
+ * Can be null.
+ * @param exitCodeCallback code to be called passing the exit code value
+ * when the process completes
+ * @param dir the working directory of the new process
+ * @return the new process
+ * @throws IOException
+ * @see java.lang.ProcessBuilder
+ */
+ public static Process launchProcess(List<String> command,
+ Map<String,String> environment,
+ final String logPrefix,
+ final ExitCodeCallback exitCodeCallback,
+ File dir)
+ throws IOException {
+ ProcessBuilder builder = new ProcessBuilder(command);
+ Map<String,String> procEnv = builder.environment();
+ if (dir != null) {
+ builder.directory(dir);
+ }
+ builder.redirectErrorStream(true);
+ if (environment != null) {
+ procEnv.putAll(environment);
+ }
+ final Process process = builder.start();
+ if (logPrefix != null || exitCodeCallback != null) {
+ Utils.asyncLoop(new Callable<Object>() {
+ public Object call() {
+ if (logPrefix != null ) {
+ Utils.readAndLogStream(logPrefix,
+ process.getInputStream());
+ }
+ if (exitCodeCallback != null) {
+ try {
+ process.waitFor();
+ exitCodeCallback.call(process.exitValue());
+ } catch (InterruptedException ie) {
+ LOG.info("{} interrupted", logPrefix);
+ exitCodeCallback.call(-1);
+ }
+ }
+ return null; // Run only once.
+ }
+ });
+ }
+ return process;
+ }
+
+ static List<ACL> supervisorZkAcls() {
+ final List<ACL> acls = new ArrayList<>();
+ acls.add(ZooDefs.Ids.CREATOR_ALL_ACL.get(0));
+ acls.add(new ACL((ZooDefs.Perms.READ ^ ZooDefs.Perms.CREATE), ZooDefs.Ids.ANYONE_ID_UNSAFE));
+ return acls;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java
new file mode 100644
index 0000000..0017092
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.supervisor.timer;
+
+import java.util.Map;
+
+import org.apache.storm.command.HealthCheck;
+import org.apache.storm.daemon.supervisor.Supervisor;
+
+public class SupervisorHealthCheck implements Runnable {
+ private final Supervisor supervisor;
+
+ public SupervisorHealthCheck(Supervisor supervisor) {
+ this.supervisor = supervisor;
+ }
+
+ @Override
+ public void run() {
+ Map<String, Object> conf = supervisor.getConf();
+ int healthCode = HealthCheck.healthCheck(conf);
+ if (healthCode != 0) {
+ supervisor.shutdownAllWorkers();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
new file mode 100644
index 0000000..34c5682
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor.timer;
+
+import org.apache.storm.Config;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.daemon.supervisor.Supervisor;
+import org.apache.storm.generated.SupervisorInfo;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SupervisorHeartbeat implements Runnable {
+
+ private final IStormClusterState stormClusterState;
+ private final String supervisorId;
+ private final Map<String, Object> conf;
+ private final Supervisor supervisor;
+
+ public SupervisorHeartbeat(Map<String, Object> conf, Supervisor supervisor) {
+ this.stormClusterState = supervisor.getStormClusterState();
+ this.supervisorId = supervisor.getId();
+ this.supervisor = supervisor;
+ this.conf = conf;
+ }
+
+ private SupervisorInfo buildSupervisorInfo(Map<String, Object> conf, Supervisor supervisor) {
+ SupervisorInfo supervisorInfo = new SupervisorInfo();
+ supervisorInfo.set_time_secs(Time.currentTimeSecs());
+ supervisorInfo.set_hostname(supervisor.getHostName());
+ supervisorInfo.set_assignment_id(supervisor.getAssignmentId());
+
+ List<Long> usedPorts = new ArrayList<>();
+ usedPorts.addAll(supervisor.getCurrAssignment().get().keySet());
+ supervisorInfo.set_used_ports(usedPorts);
+ List metaDatas = (List)supervisor.getiSupervisor().getMetadata();
+ List<Long> portList = new ArrayList<>();
+ if (metaDatas != null){
+ for (Object data : metaDatas){
+ Integer port = Utils.getInt(data);
+ if (port != null)
+ portList.add(port.longValue());
+ }
+ }
+
+ supervisorInfo.set_meta(portList);
+ supervisorInfo.set_scheduler_meta((Map<String, String>) conf.get(Config.SUPERVISOR_SCHEDULER_META));
+ supervisorInfo.set_uptime_secs(supervisor.getUpTime().upTime());
+ supervisorInfo.set_version(supervisor.getStormVersion());
+ supervisorInfo.set_resources_map(mkSupervisorCapacities(conf));
+ return supervisorInfo;
+ }
+
+ private Map<String, Double> mkSupervisorCapacities(Map conf) {
+ Map<String, Double> ret = new HashMap<String, Double>();
+ Double mem = (double) (conf.get(Config.SUPERVISOR_MEMORY_CAPACITY_MB));
+ ret.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, mem);
+ Double cpu = (double) (conf.get(Config.SUPERVISOR_CPU_CAPACITY));
+ ret.put(Config.SUPERVISOR_CPU_CAPACITY, cpu);
+ return ret;
+ }
+
+ @Override
+ public void run() {
+ SupervisorInfo supervisorInfo = buildSupervisorInfo(conf, supervisor);
+ stormClusterState.supervisorHeartbeat(supervisorId, supervisorInfo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/UpdateBlobs.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/UpdateBlobs.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/UpdateBlobs.java
new file mode 100644
index 0000000..0b6d996
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/UpdateBlobs.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor.timer;
+
+import org.apache.storm.Config;
+import org.apache.storm.daemon.supervisor.Supervisor;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.KeyNotFoundException;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.localizer.LocalResource;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.NimbusLeaderNotFoundException;
+import org.apache.storm.utils.Utils;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * downloads all blobs listed in the topology configuration for all topologies assigned to this supervisor, and creates version files with a suffix. The
+ * Runnable is intended to be run periodically by a timer, created elsewhere.
+ */
+public class UpdateBlobs implements Runnable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(UpdateBlobs.class);
+
+ private Supervisor supervisor;
+
+ public UpdateBlobs(Supervisor supervisor) {
+ this.supervisor = supervisor;
+ }
+
+ @Override
+ public void run() {
+ try {
+ Map<String, Object> conf = supervisor.getConf();
+ Set<String> downloadedStormIds = SupervisorUtils.readDownloadedTopologyIds(conf);
+ AtomicReference<Map<Long, LocalAssignment>> newAssignment = supervisor.getCurrAssignment();
+ Set<String> assignedStormIds = new HashSet<>();
+ for (LocalAssignment localAssignment : newAssignment.get().values()) {
+ assignedStormIds.add(localAssignment.get_topology_id());
+ }
+ for (String stormId : downloadedStormIds) {
+ if (assignedStormIds.contains(stormId)) {
+ String stormRoot = ConfigUtils.supervisorStormDistRoot(conf, stormId);
+ LOG.debug("Checking Blob updates for storm topology id {} With target_dir: {}", stormId, stormRoot);
+ updateBlobsForTopology(conf, stormId, supervisor.getLocalizer());
+ }
+ }
+ } catch (Exception e) {
+ if (Utils.exceptionCauseIsInstanceOf(TTransportException.class, e)) {
+ LOG.error("Network error while updating blobs, will retry again later", e);
+ } else if (Utils.exceptionCauseIsInstanceOf(NimbusLeaderNotFoundException.class, e)) {
+ LOG.error("Nimbus unavailable to update blobs, will retry again later", e);
+ } else {
+ throw Utils.wrapInRuntime(e);
+ }
+ }
+ }
+
+ /**
+ * Update each blob listed in the topology configuration if the latest version of the blob has not been downloaded.
+ *
+ * @param conf
+ * @param stormId
+ * @param localizer
+ * @throws IOException
+ */
+ private void updateBlobsForTopology(Map conf, String stormId, Localizer localizer) throws IOException {
+ Map stormConf = ConfigUtils.readSupervisorStormConf(conf, stormId);
+ Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) stormConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
+ String user = (String) stormConf.get(Config.TOPOLOGY_SUBMITTER_USER);
+ List<LocalResource> localresources = SupervisorUtils.blobstoreMapToLocalresources(blobstoreMap);
+ try {
+ localizer.updateBlobs(localresources, user);
+ } catch (AuthorizationException authExp) {
+ LOG.error("AuthorizationException error", authExp);
+ } catch (KeyNotFoundException knf) {
+ LOG.error("KeyNotFoundException error", knf);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/event/EventManager.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/event/EventManager.java b/storm-core/src/jvm/org/apache/storm/event/EventManager.java
new file mode 100644
index 0000000..64536c1
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/event/EventManager.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.event;
+
+public interface EventManager extends AutoCloseable {
+ void add(Runnable eventFn);
+
+ boolean waiting();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/event/EventManagerImp.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/event/EventManagerImp.java b/storm-core/src/jvm/org/apache/storm/event/EventManagerImp.java
new file mode 100644
index 0000000..42e6d6b
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/event/EventManagerImp.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.event;
+
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InterruptedIOException;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class EventManagerImp implements EventManager {
+ private static final Logger LOG = LoggerFactory.getLogger(EventManagerImp.class);
+
+ private AtomicInteger added;
+ private AtomicInteger processed;
+ private AtomicBoolean running;
+ private Thread runner;
+
+ private LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>();
+
+ public EventManagerImp(boolean isDaemon) {
+ added = new AtomicInteger();
+ processed = new AtomicInteger();
+ running = new AtomicBoolean(true);
+ runner = new Thread() {
+ @Override
+ public void run() {
+ while (running.get()) {
+ try {
+ Runnable r = queue.take();
+ if (r == null) {
+ return;
+ }
+
+ r.run();
+ proccessInc();
+ } catch (Throwable t) {
+ if (Utils.exceptionCauseIsInstanceOf(InterruptedIOException.class, t)) {
+ LOG.info("Event manager interrupted while doing IO");
+ } else if (Utils.exceptionCauseIsInstanceOf(InterruptedException.class, t)) {
+ LOG.info("Event manager interrupted");
+ } else {
+ LOG.error("{} Error when processing event", t);
+ Utils.exitProcess(20, "Error when processing an event");
+ }
+ }
+ }
+ }
+ };
+ runner.setDaemon(isDaemon);
+ runner.start();
+ }
+
+ public void proccessInc() {
+ processed.incrementAndGet();
+ }
+
+ @Override
+ public void add(Runnable eventFn) {
+ if (!running.get()) {
+ throw new RuntimeException("Cannot add events to a shutdown event manager");
+ }
+ added.incrementAndGet();
+ queue.add(eventFn);
+ }
+
+ @Override
+ public boolean waiting() {
+ return (Time.isThreadWaiting(runner) || (processed.get() == added.get()));
+ }
+
+ @Override
+ public void close() throws Exception {
+ running.set(false);
+ runner.interrupt();
+ runner.join();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java b/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
new file mode 100644
index 0000000..7887281
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
@@ -0,0 +1,432 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.localizer;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.JarURLConnection;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.storm.Config;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.daemon.Shutdownable;
+import org.apache.storm.daemon.supervisor.AdvancedFSOps;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * This is a wrapper around the Localizer class that provides the desired
+ * async interface to Slot.
+ */
+public class AsyncLocalizer implements ILocalizer, Shutdownable {
+ /**
+ * A future that has already completed.
+ */
+ private static class AllDoneFuture implements Future<Void> {
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ return false;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ @Override
+ public boolean isDone() {
+ return true;
+ }
+
+ @Override
+ public Void get() {
+ return null;
+ }
+
+ @Override
+ public Void get(long timeout, TimeUnit unit) {
+ return null;
+ }
+
+ }
+
+ private static final Logger LOG = LoggerFactory.getLogger(AsyncLocalizer.class);
+
+ private final Localizer _localizer;
+ private final ExecutorService _execService;
+ private final boolean _isLocalMode;
+ private final Map<String, Object> _conf;
+ private final Map<String, LocalDownloadedResource> _basicPending;
+ private final Map<String, LocalDownloadedResource> _blobPending;
+ private final AdvancedFSOps _fsOps;
+
+ private class DownloadBaseBlobsDistributed implements Callable<Void> {
+ protected final String _topologyId;
+ protected final File _stormRoot;
+
+ public DownloadBaseBlobsDistributed(String topologyId) throws IOException {
+ _topologyId = topologyId;
+ _stormRoot = new File(ConfigUtils.supervisorStormDistRoot(_conf, _topologyId));
+ }
+
+ protected void downloadBaseBlobs(File tmproot) throws Exception {
+ String stormJarKey = ConfigUtils.masterStormJarKey(_topologyId);
+ String stormCodeKey = ConfigUtils.masterStormCodeKey(_topologyId);
+ String stormConfKey = ConfigUtils.masterStormConfKey(_topologyId);
+ String jarPath = ConfigUtils.supervisorStormJarPath(tmproot.getAbsolutePath());
+ String codePath = ConfigUtils.supervisorStormCodePath(tmproot.getAbsolutePath());
+ String confPath = ConfigUtils.supervisorStormConfPath(tmproot.getAbsolutePath());
+ _fsOps.forceMkdir(tmproot);
+ _fsOps.restrictDirectoryPermissions(tmproot);
+ ClientBlobStore blobStore = Utils.getClientBlobStoreForSupervisor(_conf);
+ try {
+ Utils.downloadResourcesAsSupervisor(stormJarKey, jarPath, blobStore);
+ Utils.downloadResourcesAsSupervisor(stormCodeKey, codePath, blobStore);
+ Utils.downloadResourcesAsSupervisor(stormConfKey, confPath, blobStore);
+ } finally {
+ blobStore.shutdown();
+ }
+ Utils.extractDirFromJar(jarPath, ConfigUtils.RESOURCES_SUBDIR, tmproot);
+ }
+
+ @Override
+ public Void call() throws Exception {
+ try {
+ if (_fsOps.fileExists(_stormRoot)) {
+ if (!_fsOps.supportsAtomicDirectoryMove()) {
+ LOG.warn("{} may have partially downloaded blobs, recovering", _topologyId);
+ _fsOps.deleteIfExists(_stormRoot);
+ } else {
+ LOG.warn("{} already downloaded blobs, skipping", _topologyId);
+ return null;
+ }
+ }
+ boolean deleteAll = true;
+ String tmproot = ConfigUtils.supervisorTmpDir(_conf) + Utils.FILE_PATH_SEPARATOR + Utils.uuid();
+ File tr = new File(tmproot);
+ try {
+ downloadBaseBlobs(tr);
+ _fsOps.moveDirectoryPreferAtomic(tr, _stormRoot);
+ _fsOps.setupStormCodeDir(ConfigUtils.readSupervisorStormConf(_conf, _topologyId), _stormRoot);
+ deleteAll = false;
+ } finally {
+ if (deleteAll) {
+ LOG.warn("Failed to download basic resources for topology-id {}", _topologyId);
+ _fsOps.deleteIfExists(tr);
+ _fsOps.deleteIfExists(_stormRoot);
+ }
+ }
+ return null;
+ } catch (Exception e) {
+ LOG.warn("Caught Exception While Downloading (rethrowing)... ", e);
+ throw e;
+ }
+ }
+ }
+
+ private class DownloadBaseBlobsLocal extends DownloadBaseBlobsDistributed {
+
+ public DownloadBaseBlobsLocal(String topologyId) throws IOException {
+ super(topologyId);
+ }
+
+ @Override
+ protected void downloadBaseBlobs(File tmproot) throws Exception {
+ _fsOps.forceMkdir(tmproot);
+ String stormCodeKey = ConfigUtils.masterStormCodeKey(_topologyId);
+ String stormConfKey = ConfigUtils.masterStormConfKey(_topologyId);
+ File codePath = new File(ConfigUtils.supervisorStormCodePath(tmproot.getAbsolutePath()));
+ File confPath = new File(ConfigUtils.supervisorStormConfPath(tmproot.getAbsolutePath()));
+ BlobStore blobStore = Utils.getNimbusBlobStore(_conf, null);
+ try {
+ try (OutputStream codeOutStream = _fsOps.getOutputStream(codePath)){
+ blobStore.readBlobTo(stormCodeKey, codeOutStream, null);
+ }
+ try (OutputStream confOutStream = _fsOps.getOutputStream(confPath)) {
+ blobStore.readBlobTo(stormConfKey, confOutStream, null);
+ }
+ } finally {
+ blobStore.shutdown();
+ }
+
+ ClassLoader classloader = Thread.currentThread().getContextClassLoader();
+ String resourcesJar = AsyncLocalizer.resourcesJar();
+ URL url = classloader.getResource(ConfigUtils.RESOURCES_SUBDIR);
+
+ String targetDir = tmproot + Utils.FILE_PATH_SEPARATOR + ConfigUtils.RESOURCES_SUBDIR;
+
+ if (resourcesJar != null) {
+ LOG.info("Extracting resources from jar at {} to {}", resourcesJar, targetDir);
+ Utils.extractDirFromJar(resourcesJar, ConfigUtils.RESOURCES_SUBDIR, _stormRoot);
+ } else if (url != null) {
+ LOG.info("Copying resources at {} to {} ", url.toString(), targetDir);
+ if ("jar".equals(url.getProtocol())) {
+ JarURLConnection urlConnection = (JarURLConnection) url.openConnection();
+ Utils.extractDirFromJar(urlConnection.getJarFileURL().getFile(), ConfigUtils.RESOURCES_SUBDIR, _stormRoot);
+ } else {
+ _fsOps.copyDirectory(new File(url.getFile()), new File(targetDir));
+ }
+ }
+ }
+ }
+
+ private class DownloadBlobs implements Callable<Void> {
+ private final String _topologyId;
+
+ public DownloadBlobs(String topologyId) {
+ _topologyId = topologyId;
+ }
+
+ @Override
+ public Void call() throws Exception {
+ try {
+ String stormroot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId);
+ Map<String, Object> stormConf = ConfigUtils.readSupervisorStormConf(_conf, _topologyId);
+
+ @SuppressWarnings("unchecked")
+ Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) stormConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
+ String user = (String) stormConf.get(Config.TOPOLOGY_SUBMITTER_USER);
+ String topoName = (String) stormConf.get(Config.TOPOLOGY_NAME);
+
+ List<LocalResource> localResourceList = new ArrayList<>();
+ if (blobstoreMap != null) {
+ List<LocalResource> tmp = SupervisorUtils.blobstoreMapToLocalresources(blobstoreMap);
+ if (tmp != null) {
+ localResourceList.addAll(tmp);
+ }
+ }
+
+ StormTopology stormCode = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _fsOps);
+ List<String> dependencies = new ArrayList<>();
+ if (stormCode.is_set_dependency_jars()) {
+ dependencies.addAll(stormCode.get_dependency_jars());
+ }
+ if (stormCode.is_set_dependency_artifacts()) {
+ dependencies.addAll(stormCode.get_dependency_artifacts());
+ }
+ for (String dependency : dependencies) {
+ localResourceList.add(new LocalResource(dependency, false));
+ }
+
+ if (!localResourceList.isEmpty()) {
+ File userDir = _localizer.getLocalUserFileCacheDir(user);
+ if (!_fsOps.fileExists(userDir)) {
+ _fsOps.forceMkdir(userDir);
+ }
+ List<LocalizedResource> localizedResources = _localizer.getBlobs(localResourceList, user, topoName, userDir);
+ _fsOps.setupBlobPermissions(userDir, user);
+ for (LocalizedResource localizedResource : localizedResources) {
+ String keyName = localizedResource.getKey();
+ //The sym link we are pointing to
+ File rsrcFilePath = new File(localizedResource.getCurrentSymlinkPath());
+
+ String symlinkName = null;
+ if (blobstoreMap != null) {
+ Map<String, Object> blobInfo = blobstoreMap.get(keyName);
+ if (blobInfo != null && blobInfo.containsKey("localname")) {
+ symlinkName = (String) blobInfo.get("localname");
+ } else {
+ symlinkName = keyName;
+ }
+ } else {
+ // all things are from dependencies
+ symlinkName = keyName;
+ }
+ _fsOps.createSymlink(new File(stormroot, symlinkName), rsrcFilePath);
+ }
+ }
+
+ return null;
+ } catch (Exception e) {
+ LOG.warn("Caught Exception While Downloading (rethrowing)... ", e);
+ throw e;
+ }
+ }
+ }
+
+ //Visible for testing
+ AsyncLocalizer(Map<String, Object> conf, Localizer localizer, AdvancedFSOps ops) {
+ _conf = conf;
+ _isLocalMode = ConfigUtils.isLocalMode(conf);
+ _localizer = localizer;
+ _execService = Executors.newFixedThreadPool(1,
+ new ThreadFactoryBuilder()
+ .setNameFormat("Async Localizer")
+ .build());
+ _basicPending = new HashMap<>();
+ _blobPending = new HashMap<>();
+ _fsOps = ops;
+ }
+
+ public AsyncLocalizer(Map<String, Object> conf, Localizer localizer) {
+ this(conf, localizer, AdvancedFSOps.make(conf));
+ }
+
+ @Override
+ public synchronized Future<Void> requestDownloadBaseTopologyBlobs(final LocalAssignment assignment, final int port) throws IOException {
+ final String topologyId = assignment.get_topology_id();
+ LocalDownloadedResource localResource = _basicPending.get(topologyId);
+ if (localResource == null) {
+ Callable<Void> c;
+ if (_isLocalMode) {
+ c = new DownloadBaseBlobsLocal(topologyId);
+ } else {
+ c = new DownloadBaseBlobsDistributed(topologyId);
+ }
+ localResource = new LocalDownloadedResource(_execService.submit(c));
+ _basicPending.put(topologyId, localResource);
+ }
+ Future<Void> ret = localResource.reserve(port, assignment);
+ LOG.debug("Reserved basic {} {}", topologyId, localResource);
+ return ret;
+ }
+
+ private static String resourcesJar() throws IOException {
+ String path = Utils.currentClasspath();
+ if (path == null) {
+ return null;
+ }
+
+ for (String jpath : path.split(File.pathSeparator)) {
+ if (jpath.endsWith(".jar")) {
+ if (Utils.zipDoesContainDir(jpath, ConfigUtils.RESOURCES_SUBDIR)) {
+ return jpath;
+ }
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public synchronized void recoverRunningTopology(LocalAssignment assignment, int port) {
+ final String topologyId = assignment.get_topology_id();
+ LocalDownloadedResource localResource = _basicPending.get(topologyId);
+ if (localResource == null) {
+ localResource = new LocalDownloadedResource(new AllDoneFuture());
+ _basicPending.put(topologyId, localResource);
+ }
+ localResource.reserve(port, assignment);
+ LOG.debug("Recovered basic {} {}", topologyId, localResource);
+
+ localResource = _blobPending.get(topologyId);
+ if (localResource == null) {
+ localResource = new LocalDownloadedResource(new AllDoneFuture());
+ _blobPending.put(topologyId, localResource);
+ }
+ localResource.reserve(port, assignment);
+ LOG.debug("Recovered blobs {} {}", topologyId, localResource);
+ }
+
+ @Override
+ public synchronized Future<Void> requestDownloadTopologyBlobs(LocalAssignment assignment, int port) {
+ final String topologyId = assignment.get_topology_id();
+ LocalDownloadedResource localResource = _blobPending.get(topologyId);
+ if (localResource == null) {
+ Callable<Void> c = new DownloadBlobs(topologyId);
+ localResource = new LocalDownloadedResource(_execService.submit(c));
+ _blobPending.put(topologyId, localResource);
+ }
+ Future<Void> ret = localResource.reserve(port, assignment);
+ LOG.debug("Reserved blobs {} {}", topologyId, localResource);
+ return ret;
+ }
+
+ @Override
+ public synchronized void releaseSlotFor(LocalAssignment assignment, int port) throws IOException {
+ final String topologyId = assignment.get_topology_id();
+ LOG.debug("Releasing slot for {} {}", topologyId, port);
+ LocalDownloadedResource localResource = _blobPending.get(topologyId);
+ if (localResource == null || !localResource.release(port, assignment)) {
+ LOG.warn("Released blob reference {} {} for something that we didn't have {}", topologyId, port, localResource);
+ } else if (localResource.isDone()){
+ LOG.info("Released blob reference {} {} Cleaning up BLOB references...", topologyId, port);
+ _blobPending.remove(topologyId);
+ Map<String, Object> topoConf = ConfigUtils.readSupervisorStormConf(_conf, topologyId);
+ @SuppressWarnings("unchecked")
+ Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
+ if (blobstoreMap != null) {
+ String user = (String) topoConf.get(Config.TOPOLOGY_SUBMITTER_USER);
+ String topoName = (String) topoConf.get(Config.TOPOLOGY_NAME);
+
+ for (Map.Entry<String, Map<String, Object>> entry : blobstoreMap.entrySet()) {
+ String key = entry.getKey();
+ Map<String, Object> blobInfo = entry.getValue();
+ try {
+ _localizer.removeBlobReference(key, user, topoName, SupervisorUtils.shouldUncompressBlob(blobInfo));
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+ }
+ } else {
+ LOG.debug("Released blob reference {} {} still waiting on {}", topologyId, port, localResource);
+ }
+
+ localResource = _basicPending.get(topologyId);
+ if (localResource == null || !localResource.release(port, assignment)) {
+ LOG.warn("Released basic reference {} {} for something that we didn't have {}", topologyId, port, localResource);
+ } else if (localResource.isDone()){
+ LOG.info("Released blob reference {} {} Cleaning up basic files...", topologyId, port);
+ _basicPending.remove(topologyId);
+ String path = ConfigUtils.supervisorStormDistRoot(_conf, topologyId);
+ _fsOps.deleteIfExists(new File(path), null, "rmr "+topologyId);
+ } else {
+ LOG.debug("Released basic reference {} {} still waiting on {}", topologyId, port, localResource);
+ }
+ }
+
+ @Override
+ public synchronized void cleanupUnusedTopologies() throws IOException {
+ File distRoot = new File(ConfigUtils.supervisorStormDistRoot(_conf));
+ LOG.info("Cleaning up unused topologies in {}", distRoot);
+ File[] children = distRoot.listFiles();
+ if (children != null) {
+ for (File topoDir : children) {
+ String topoId = URLDecoder.decode(topoDir.getName(), "UTF-8");
+ if (_basicPending.get(topoId) == null && _blobPending.get(topoId) == null) {
+ _fsOps.deleteIfExists(topoDir, null, "rmr " + topoId);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void shutdown() {
+ _execService.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/ILocalizer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/ILocalizer.java b/storm-core/src/jvm/org/apache/storm/localizer/ILocalizer.java
new file mode 100644
index 0000000..7105095
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/localizer/ILocalizer.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.localizer;
+
+import java.io.IOException;
+import java.util.concurrent.Future;
+
+import org.apache.storm.generated.LocalAssignment;
+
+/**
+ * Download blobs from the blob store and keep them up to date.
+ */
+public interface ILocalizer {
+
+ /**
+ * Recover a running topology by incrementing references for what it has already downloaded.
+ * @param assignment the assignment the resources are for
+ * @param port the port the topology is running in.
+ */
+ void recoverRunningTopology(LocalAssignment assignemnt, int port);
+
+ /**
+ * Download storm.jar, storm.conf, and storm.ser for this topology if not done so already,
+ * and inc a reference count on them.
+ * @param assignment the assignment the resources are for
+ * @param port the port the topology is running on
+ * @return a future to let you know when they are done.
+ * @throws IOException on error
+ */
+ Future<Void> requestDownloadBaseTopologyBlobs(LocalAssignment assignment, int port) throws IOException;
+
+ /**
+ * Download the blobs for this topology (reading in list in from the config)
+ * and inc reference count for these blobs.
+ * PRECONDITION: requestDownloadBaseTopologyBlobs has completed downloading.
+ * @param assignment the assignment the resources are for
+ * @param port the port the topology is running on
+ * @return a future to let you know when they are done.
+ */
+ Future<Void> requestDownloadTopologyBlobs(LocalAssignment assignment, int port);
+
+ /**
+ * dec reference count on all blobs associated with this topology.
+ * @param assignment the assignment the resources are for
+ * @param port the port the topology is running on
+ * @throws IOException on any error
+ */
+ void releaseSlotFor(LocalAssignment assignment, int port) throws IOException;
+
+ /**
+ * Clean up any topologies that are not in use right now.
+ * @throws IOException on any error.
+ */
+ void cleanupUnusedTopologies() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/LocalDownloadedResource.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/LocalDownloadedResource.java b/storm-core/src/jvm/org/apache/storm/localizer/LocalDownloadedResource.java
new file mode 100644
index 0000000..570c951
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/localizer/LocalDownloadedResource.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.localizer;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.storm.generated.LocalAssignment;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LocalDownloadedResource {
+ private static final Logger LOG = LoggerFactory.getLogger(LocalDownloadedResource.class);
+ private static class NoCancelFuture<T> implements Future<T> {
+ private final Future<T> _wrapped;
+
+ public NoCancelFuture(Future<T> wrapped) {
+ _wrapped = wrapped;
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ //cancel not currently supported
+ return false;
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ @Override
+ public boolean isDone() {
+ return _wrapped.isDone();
+ }
+
+ @Override
+ public T get() throws InterruptedException, ExecutionException {
+ return _wrapped.get();
+ }
+
+ @Override
+ public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+ return _wrapped.get(timeout, unit);
+ }
+ }
+ private static class PortNAssignment {
+ private final int _port;
+ private final LocalAssignment _assignment;
+
+ public PortNAssignment(int port, LocalAssignment assignment) {
+ _port = port;
+ _assignment = assignment;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof PortNAssignment)) {
+ return false;
+ }
+ PortNAssignment pna = (PortNAssignment) other;
+ return pna._port == _port && _assignment.equals(pna._assignment);
+ }
+
+ @Override
+ public int hashCode() {
+ return (17 * _port) + _assignment.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "{"+ _port + " " + _assignment +"}";
+ }
+ }
+ private final Future<Void> _pending;
+ private final Set<PortNAssignment> _references;
+ private boolean _isDone;
+
+
+ public LocalDownloadedResource(Future<Void> pending) {
+ _pending = new NoCancelFuture<>(pending);
+ _references = new HashSet<>();
+ _isDone = false;
+ }
+
+ /**
+ * Reserve the resources
+ * @param port the port this is for
+ * @param la the assignment this is for
+ * @return a future that can be used to track it being downloaded.
+ */
+ public synchronized Future<Void> reserve(int port, LocalAssignment la) {
+ PortNAssignment pna = new PortNAssignment(port, la);
+ if (!_references.add(pna)) {
+ LOG.warn("Resources {} already reserved {} for this topology", pna, _references);
+ }
+ return _pending;
+ }
+
+ /**
+ * Release a port from the reference count, and update isDone if all is done.
+ * @param port the port to release
+ * @param la the assignment to release
+ * @return true if the port was being counted else false
+ */
+ public synchronized boolean release(int port, LocalAssignment la) {
+ PortNAssignment pna = new PortNAssignment(port, la);
+ boolean ret = _references.remove(pna);
+ if (ret && _references.isEmpty()) {
+ _isDone = true;
+ }
+ return ret;
+ }
+
+ /**
+ * Is this has been cleaned up completely.
+ * @return true if it is done else false
+ */
+ public synchronized boolean isDone() {
+ return _isDone;
+ }
+
+ @Override
+ public String toString() {
+ return _references.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSet.java b/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
index 380e777..9f42b47 100644
--- a/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
+++ b/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
@@ -93,7 +93,7 @@ public class LocalizedResourceRetentionSet {
i.remove();
} else {
// since it failed to delete add it back so it gets retried
- set.addResource(resource.getKey(), resource, resource.isUncompressed());
+ set.add(resource.getKey(), resource, resource.isUncompressed());
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceSet.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceSet.java b/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceSet.java
index b5f00c3..62d5b2f 100644
--- a/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceSet.java
+++ b/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceSet.java
@@ -57,7 +57,7 @@ public class LocalizedResourceSet {
return _localrsrcFiles.get(name);
}
- public void updateResource(String resourceName, LocalizedResource updatedResource,
+ public void putIfAbsent(String resourceName, LocalizedResource updatedResource,
boolean uncompress) {
if (uncompress) {
_localrsrcArchives.putIfAbsent(resourceName, updatedResource);
@@ -66,7 +66,7 @@ public class LocalizedResourceSet {
}
}
- public void addResource(String resourceName, LocalizedResource newResource, boolean uncompress) {
+ public void add(String resourceName, LocalizedResource newResource, boolean uncompress) {
if (uncompress) {
_localrsrcArchives.put(resourceName, newResource);
} else {
@@ -76,9 +76,9 @@ public class LocalizedResourceSet {
public boolean exists(String resourceName, boolean uncompress) {
if (uncompress) {
- return (_localrsrcArchives.get(resourceName) != null);
+ return _localrsrcArchives.containsKey(resourceName);
}
- return (_localrsrcFiles.get(resourceName) != null);
+ return _localrsrcFiles.containsKey(resourceName);
}
public boolean remove(LocalizedResource resource) {
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/Localizer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/Localizer.java b/storm-core/src/jvm/org/apache/storm/localizer/Localizer.java
index b91cecb..0135397 100644
--- a/storm-core/src/jvm/org/apache/storm/localizer/Localizer.java
+++ b/storm-core/src/jvm/org/apache/storm/localizer/Localizer.java
@@ -63,20 +63,6 @@ import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
*/
public class Localizer {
public static final Logger LOG = LoggerFactory.getLogger(Localizer.class);
-
- private Map _conf;
- private int _threadPoolSize;
- // thread pool for initial download
- private ExecutorService _execService;
- // thread pool for updates
- private ExecutorService _updateExecService;
- private int _blobDownloadRetries;
-
- // track resources - user to resourceSet
- private final ConcurrentMap<String, LocalizedResourceSet> _userRsrc = new
- ConcurrentHashMap<String, LocalizedResourceSet>();
-
- private String _localBaseDir;
public static final String USERCACHE = "usercache";
public static final String FILECACHE = "filecache";
@@ -85,13 +71,29 @@ public class Localizer {
public static final String ARCHIVESDIR = "archives";
private static final String TO_UNCOMPRESS = "_tmp_";
+
+
+
+ private final Map<String, Object> _conf;
+ private final int _threadPoolSize;
+ // thread pool for initial download
+ private final ExecutorService _execService;
+ // thread pool for updates
+ private final ExecutorService _updateExecService;
+ private final int _blobDownloadRetries;
+
+ // track resources - user to resourceSet
+ private final ConcurrentMap<String, LocalizedResourceSet> _userRsrc = new
+ ConcurrentHashMap<String, LocalizedResourceSet>();
+
+ private final String _localBaseDir;
// cleanup
private long _cacheTargetSize;
private long _cacheCleanupPeriod;
private ScheduledExecutorService _cacheCleanupService;
- public Localizer(Map conf, String baseDir) {
+ public Localizer(Map<String, Object> conf, String baseDir) {
_conf = conf;
_localBaseDir = baseDir;
// default cache size 10GB, converted to Bytes
@@ -189,7 +191,7 @@ public class Localizer {
LOG.debug("local file is: {} path is: {}", rsrc.getPath(), path);
LocalizedResource lrsrc = new LocalizedResource(new File(path).getName(), path,
uncompress);
- lrsrcSet.addResource(lrsrc.getKey(), lrsrc, uncompress);
+ lrsrcSet.add(lrsrc.getKey(), lrsrc, uncompress);
}
}
}
@@ -369,7 +371,7 @@ public class Localizer {
if (newlrsrcSet == null) {
newlrsrcSet = newSet;
}
- newlrsrcSet.updateResource(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
+ newlrsrcSet.putIfAbsent(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
results.add(lrsrc);
}
catch (ExecutionException e) {
@@ -451,7 +453,7 @@ public class Localizer {
for (Future<LocalizedResource> futureRsrc: futures) {
LocalizedResource lrsrc = futureRsrc.get();
lrsrc.addReference(topo);
- lrsrcSet.addResource(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
+ lrsrcSet.add(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
results.add(lrsrc);
}
} catch (ExecutionException e) {
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/metric/StormMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/metric/StormMetricsRegistry.java b/storm-core/src/jvm/org/apache/storm/metric/StormMetricsRegistry.java
new file mode 100644
index 0000000..d433920
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/metric/StormMetricsRegistry.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.metric;
+
+import com.codahale.metrics.*;
+
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.apache.storm.daemon.metrics.MetricsUtils;
+import org.apache.storm.daemon.metrics.reporters.PreparableReporter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("unchecked")
+public class StormMetricsRegistry {
+ private static final Logger LOG = LoggerFactory.getLogger(StormMetricsRegistry.class);
+ public static final MetricRegistry DEFAULT_REGISTRY = new MetricRegistry();
+
+ public static Meter registerMeter(String name) {
+ Meter meter = new Meter();
+ return register(name, meter);
+ }
+
+ // TODO: should replace Callable to Gauge<Integer> when nimbus.clj is translated to java
+ public static Gauge<Integer> registerGauge(final String name, final Callable fn) {
+ Gauge<Integer> gauge = new Gauge<Integer>() {
+ @Override
+ public Integer getValue() {
+ try {
+ return (Integer) fn.call();
+ } catch (Exception e) {
+ LOG.error("Error getting gauge value for {}", name, e);
+ }
+ return 0;
+ }
+ };
+ return register(name, gauge);
+ }
+
+ public static Histogram registerHistogram(String name, Reservoir reservoir) {
+ Histogram histogram = new Histogram(reservoir);
+ return register(name, histogram);
+ }
+
+ public static void startMetricsReporters(Map stormConf) {
+ for (PreparableReporter reporter : MetricsUtils.getPreparableReporters(stormConf)) {
+ startMetricsReporter(reporter, stormConf);
+ }
+ }
+
+ private static void startMetricsReporter(PreparableReporter reporter, Map stormConf) {
+ reporter.prepare(StormMetricsRegistry.DEFAULT_REGISTRY, stormConf);
+ reporter.start();
+ LOG.info("Started statistics report plugin...");
+ }
+
+ private static <T extends Metric> T register(String name, T metric) {
+ T ret;
+ try {
+ ret = DEFAULT_REGISTRY.register(name, metric);
+ } catch (IllegalArgumentException e) {
+ // swallow IllegalArgumentException when the metric exists already
+ ret = (T) DEFAULT_REGISTRY.getMetrics().get(name);
+ if (ret == null) {
+ throw e;
+ } else {
+ LOG.warn("Metric {} has already been registered", name);
+ }
+ }
+ return ret;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/nimbus/ILeaderElector.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/nimbus/ILeaderElector.java b/storm-core/src/jvm/org/apache/storm/nimbus/ILeaderElector.java
index 3c729ec..e8789df 100644
--- a/storm-core/src/jvm/org/apache/storm/nimbus/ILeaderElector.java
+++ b/storm-core/src/jvm/org/apache/storm/nimbus/ILeaderElector.java
@@ -37,14 +37,14 @@ public interface ILeaderElector extends Closeable {
* check isLeader() to perform any leadership action. This method can be called
* multiple times so it needs to be idempotent.
*/
- void addToLeaderLockQueue();
+ void addToLeaderLockQueue() throws Exception;
/**
* Removes the caller from the leader lock queue. If the caller is leader
* also releases the lock. This method can be called multiple times so it needs
* to be idempotent.
*/
- void removeFromLeaderLockQueue();
+ void removeFromLeaderLockQueue() throws Exception;
/**
*
@@ -62,7 +62,7 @@ public interface ILeaderElector extends Closeable {
*
* @return list of current nimbus addresses, includes leader.
*/
- List<NimbusInfo> getAllNimbuses();
+ List<NimbusInfo> getAllNimbuses()throws Exception;
/**
* Method called to allow for cleanup. once close this object can not be reused.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java b/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
index 32d28fe..8415ce3 100644
--- a/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
+++ b/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
@@ -134,6 +134,21 @@ public class SerializationFactory {
Map<String, Map<String, Integer>> streamNametoId = new HashMap<>();
Map<String, Map<Integer, String>> streamIdToName = new HashMap<>();
+ /**
+ * "{:a 1 :b 2} -> {1 :a 2 :b}"
+ *
+ * Note: Only one key wins if there are duplicate values.
+ * Which key wins is indeterminate:
+ * "{:a 1 :b 1} -> {1 :a} *or* {1 :b}"
+ */
+ private static <K, V> Map<V, K> simpleReverseMap(Map<K, V> map) {
+ Map<V, K> ret = new HashMap<V, K>();
+ for (Map.Entry<K, V> entry : map.entrySet()) {
+ ret.put(entry.getValue(), entry.getKey());
+ }
+ return ret;
+ }
+
public IdDictionary(StormTopology topology) {
List<String> componentNames = new ArrayList<>(topology.get_spouts().keySet());
componentNames.addAll(topology.get_bolts().keySet());
@@ -143,7 +158,7 @@ public class SerializationFactory {
ComponentCommon common = Utils.getComponentCommon(topology, name);
List<String> streams = new ArrayList<>(common.get_streams().keySet());
streamNametoId.put(name, idify(streams));
- streamIdToName.put(name, Utils.reverseMap(streamNametoId.get(name)));
+ streamIdToName.put(name, simpleReverseMap(streamNametoId.get(name)));
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java b/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
index 2f06102..0284725 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
@@ -17,17 +17,18 @@
*/
package org.apache.storm.testing;
-import org.apache.storm.topology.OutputFieldsDeclarer;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.UUID;
+
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.utils.InprocMessaging;
-import java.util.HashMap;
-import java.util.List;
-import java.util.UUID;
public class FeederSpout extends BaseRichSpout {
@@ -51,7 +52,15 @@ public class FeederSpout extends BaseRichSpout {
public void feed(List<Object> tuple, Object msgId) {
InprocMessaging.sendMessage(_id, new Values(tuple, msgId));
- }
+ }
+
+ public void feedNoWait(List<Object> tuple, Object msgId) {
+ InprocMessaging.sendMessageNoWait(_id, new Values(tuple, msgId));
+ }
+
+ public void waitForReader() {
+ InprocMessaging.waitForReader(_id);
+ }
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
_collector = collector;
@@ -63,17 +72,11 @@ public class FeederSpout extends BaseRichSpout {
public void nextTuple() {
List<Object> toEmit = (List<Object>) InprocMessaging.pollMessage(_id);
- if(toEmit!=null) {
+ if (toEmit!=null) {
List<Object> tuple = (List<Object>) toEmit.get(0);
Object msgId = toEmit.get(1);
_collector.emit(tuple, msgId);
- } else {
- try {
- Thread.sleep(1);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
}
}
[07/21] storm git commit: STORM-2018: Supervisor V2
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/ZKStateStorage.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/ZKStateStorage.java b/storm-core/src/jvm/org/apache/storm/cluster/ZKStateStorage.java
new file mode 100644
index 0000000..e337b1f
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/ZKStateStorage.java
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.state.*;
+import org.apache.curator.framework.state.ConnectionState;
+import org.apache.storm.Config;
+import org.apache.storm.callback.DefaultWatcherCallBack;
+import org.apache.storm.callback.WatcherCallBack;
+import org.apache.storm.callback.ZKStateChangedCallback;
+import org.apache.storm.utils.Utils;
+import org.apache.storm.zookeeper.Zookeeper;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.ACL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class ZKStateStorage implements IStateStorage {
+
+ private static Logger LOG = LoggerFactory.getLogger(ZKStateStorage.class);
+
+ private ConcurrentHashMap<String, ZKStateChangedCallback> callbacks = new ConcurrentHashMap<String, ZKStateChangedCallback>();
+ private CuratorFramework zkWriter;
+ private CuratorFramework zkReader;
+ private AtomicBoolean active;
+
+ private boolean isNimbus;
+ private Map authConf;
+ private Map<Object, Object> conf;
+
+ private class ZkWatcherCallBack implements WatcherCallBack{
+ @Override
+ public void execute(Watcher.Event.KeeperState state, Watcher.Event.EventType type, String path) {
+ if (active.get()) {
+ if (!(state.equals(Watcher.Event.KeeperState.SyncConnected))) {
+ LOG.debug("Received event {} : {}: {} with disconnected Zookeeper.", state, type, path);
+ } else {
+ LOG.debug("Received event {} : {} : {}", state, type, path);
+ }
+
+ if (!type.equals(Watcher.Event.EventType.None)) {
+ for (Map.Entry<String, ZKStateChangedCallback> e : callbacks.entrySet()) {
+ ZKStateChangedCallback fn = e.getValue();
+ fn.changed(type, path);
+ }
+ }
+ }
+ }
+ }
+
+ public ZKStateStorage(Map<Object, Object> conf, Map authConf, List<ACL> acls, ClusterStateContext context) throws Exception {
+ this.conf = conf;
+ this.authConf = authConf;
+ if (context.getDaemonType().equals(DaemonType.NIMBUS))
+ this.isNimbus = true;
+
+ // just mkdir STORM_ZOOKEEPER_ROOT dir
+ CuratorFramework zkTemp = mkZk();
+ String rootPath = String.valueOf(conf.get(Config.STORM_ZOOKEEPER_ROOT));
+ Zookeeper.mkdirs(zkTemp, rootPath, acls);
+ zkTemp.close();
+
+ active = new AtomicBoolean(true);
+ zkWriter = mkZk(new ZkWatcherCallBack());
+ if (isNimbus) {
+ zkReader = mkZk(new ZkWatcherCallBack());
+ } else {
+ zkReader = zkWriter;
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ private CuratorFramework mkZk() throws IOException {
+ return Zookeeper.mkClient(conf, (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS), conf.get(Config.STORM_ZOOKEEPER_PORT), "",
+ new DefaultWatcherCallBack(), authConf);
+ }
+
+ @SuppressWarnings("unchecked")
+ private CuratorFramework mkZk(WatcherCallBack watcher) throws NumberFormatException, IOException {
+ return Zookeeper.mkClient(conf, (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS), conf.get(Config.STORM_ZOOKEEPER_PORT),
+ String.valueOf(conf.get(Config.STORM_ZOOKEEPER_ROOT)), watcher, authConf);
+ }
+
+ @Override
+ public void delete_node_blobstore(String path, String nimbusHostPortInfo) {
+ Zookeeper.deleteNodeBlobstore(zkWriter, path, nimbusHostPortInfo);
+ }
+
+ @Override
+ public String register(ZKStateChangedCallback callback) {
+ String id = UUID.randomUUID().toString();
+ this.callbacks.put(id, callback);
+ return id;
+ }
+
+ @Override
+ public void unregister(String id) {
+ this.callbacks.remove(id);
+ }
+
+ @Override
+ public String create_sequential(String path, byte[] data, List<ACL> acls) {
+ return Zookeeper.createNode(zkWriter, path, data, CreateMode.EPHEMERAL_SEQUENTIAL, acls);
+ }
+
+ @Override
+ public void mkdirs(String path, List<ACL> acls) {
+ Zookeeper.mkdirs(zkWriter, path, acls);
+ }
+
+ @Override
+ public void delete_node(String path) {
+ Zookeeper.deleteNode(zkWriter, path);
+ }
+
+ @Override
+ public void set_ephemeral_node(String path, byte[] data, List<ACL> acls) {
+ Zookeeper.mkdirs(zkWriter, Zookeeper.parentPath(path), acls);
+ if (Zookeeper.exists(zkWriter, path, false)) {
+ try {
+ Zookeeper.setData(zkWriter, path, data);
+ } catch (RuntimeException e) {
+ if (Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) {
+ Zookeeper.createNode(zkWriter, path, data, CreateMode.EPHEMERAL, acls);
+ } else {
+ throw e;
+ }
+ }
+
+ } else {
+ Zookeeper.createNode(zkWriter, path, data, CreateMode.EPHEMERAL, acls);
+ }
+ }
+
+ @Override
+ public Integer get_version(String path, boolean watch) throws Exception {
+ Integer ret = Zookeeper.getVersion(zkReader, path, watch);
+ return ret;
+ }
+
+ @Override
+ public boolean node_exists(String path, boolean watch) {
+ return Zookeeper.existsNode(zkReader, path, watch);
+ }
+
+ @Override
+ public List<String> get_children(String path, boolean watch) {
+ return Zookeeper.getChildren(zkReader, path, watch);
+ }
+
+ @Override
+ public void close() {
+ this.active.set(false);
+ zkWriter.close();
+ if (isNimbus) {
+ zkReader.close();
+ }
+ }
+
+ @Override
+ public void set_data(String path, byte[] data, List<ACL> acls) {
+ if (Zookeeper.exists(zkWriter, path, false)) {
+ Zookeeper.setData(zkWriter, path, data);
+ } else {
+ Zookeeper.mkdirs(zkWriter, Zookeeper.parentPath(path), acls);
+ try {
+ Zookeeper.createNode(zkWriter, path, data, CreateMode.PERSISTENT, acls);
+ } catch (RuntimeException e) {
+ if (Utils.exceptionCauseIsInstanceOf(KeeperException.NodeExistsException.class, e)) {
+ Zookeeper.setData(zkWriter, path, data);
+ } else {
+ throw e;
+ }
+ }
+ }
+ }
+
+ @Override
+ public byte[] get_data(String path, boolean watch) {
+ byte[] ret = null;
+
+ ret = Zookeeper.getData(zkReader, path, watch);
+
+ return ret;
+ }
+
+ @Override
+ public VersionedData<byte[]> get_data_with_version(String path, boolean watch) {
+ return Zookeeper.getDataWithVersion(zkReader, path, watch);
+ }
+
+ @Override
+ public void set_worker_hb(String path, byte[] data, List<ACL> acls) {
+ set_data(path, data, acls);
+ }
+
+ @Override
+ public byte[] get_worker_hb(String path, boolean watch) {
+ return Zookeeper.getData(zkReader, path, watch);
+ }
+
+ @Override
+ public List<String> get_worker_hb_children(String path, boolean watch) {
+ return get_children(path, watch);
+ }
+
+ @Override
+ public void delete_worker_hb(String path) {
+ delete_node(path);
+ }
+
+ @Override
+ public void add_listener(final ConnectionStateListener listener) {
+ Zookeeper.addListener(zkReader, new ConnectionStateListener() {
+ @Override
+ public void stateChanged(CuratorFramework curatorFramework, ConnectionState connectionState) {
+ listener.stateChanged(curatorFramework, connectionState);
+ }
+ });
+ }
+
+ @Override
+ public void sync_path(String path) {
+ Zookeeper.syncPath(zkWriter, path);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java b/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java
new file mode 100644
index 0000000..73bcea0
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.command;
+
+import org.apache.storm.Config;
+import org.apache.storm.utils.Utils;
+
+import org.apache.storm.utils.ConfigUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class HealthCheck {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HealthCheck.class);
+ private static final String FAILED = "failed";
+ private static final String SUCCESS = "success";
+ private static final String TIMEOUT = "timeout";
+ private static final String FAILED_WITH_EXIT_CODE = "failed_with_exit_code";
+
+ public static int healthCheck(Map conf) {
+ String healthDir = ConfigUtils.absoluteHealthCheckDir(conf);
+ List<String> results = new ArrayList<>();
+ if (healthDir != null) {
+ File parentFile = new File(healthDir);
+ List<String> healthScripts = new ArrayList<String>();
+ if (parentFile.exists()) {
+ File[] list = parentFile.listFiles();
+ for (File f : list) {
+ if (!f.isDirectory() && f.canExecute())
+ healthScripts.add(f.getAbsolutePath());
+ }
+ }
+ for (String script : healthScripts) {
+ String result = processScript(conf, script);
+ results.add(result);
+ }
+ }
+
+ // failed_with_exit_code is OK. We're mimicing Hadoop's health checks.
+ // We treat non-zero exit codes as indicators that the scripts failed
+ // to execute properly, not that the system is unhealthy, in which case
+ // we don't want to start killing things.
+
+ if (results.contains(FAILED) || results.contains(TIMEOUT)) {
+ return 1;
+ } else {
+ return 0;
+ }
+
+ }
+
+ public static String processScript(Map conf, String script) {
+ Thread interruptThread = null;
+ try {
+ Process process = Runtime.getRuntime().exec(script);
+ final long timeout = (long) (conf.get(Config.STORM_HEALTH_CHECK_TIMEOUT_MS));
+ final Thread curThread = Thread.currentThread();
+ // kill process when timeout
+ interruptThread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(timeout);
+ curThread.interrupt();
+ } catch (InterruptedException e) {
+ // Ignored
+ }
+ }
+ });
+ interruptThread.start();
+ process.waitFor();
+ interruptThread.interrupt();
+ curThread.interrupted();
+
+ if (process.exitValue() != 0) {
+ String str;
+ InputStream stdin = process.getInputStream();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(stdin));
+ while ((str = reader.readLine()) != null) {
+ if (str.startsWith("ERROR")) {
+ return FAILED;
+ }
+ }
+ return SUCCESS;
+ }
+ return FAILED_WITH_EXIT_CODE;
+ } catch (InterruptedException e) {
+ LOG.warn("Script: {} timed out.", script);
+ return TIMEOUT;
+ } catch (Exception e) {
+ LOG.warn("Script failed with exception: ", e);
+ return FAILED_WITH_EXIT_CODE;
+ } finally {
+ if (interruptThread != null)
+ interruptThread.interrupt();
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ Map<String, Object> conf = ConfigUtils.readStormConfig();
+ System.exit(healthCheck(conf));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java b/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java
new file mode 100644
index 0000000..1fcd81b
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.command;
+
+import java.io.File;
+import java.util.Map;
+
+import org.apache.storm.Config;
+import org.apache.storm.utils.Utils;
+
+import org.apache.storm.daemon.supervisor.StandaloneSupervisor;
+import org.apache.storm.daemon.supervisor.Supervisor;
+import org.apache.storm.utils.ConfigUtils;
+
+public class KillWorkers {
+ public static void main(String [] args) throws Exception {
+ Map<String, Object> conf = ConfigUtils.readStormConfig();
+ conf.put(Config.STORM_LOCAL_DIR, new File((String)conf.get(Config.STORM_LOCAL_DIR)).getCanonicalPath());
+ try (Supervisor supervisor = new Supervisor(conf, null, new StandaloneSupervisor())) {
+ supervisor.shutdownAllWorkers();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/DaemonCommon.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/DaemonCommon.java b/storm-core/src/jvm/org/apache/storm/daemon/DaemonCommon.java
new file mode 100644
index 0000000..d1b71a7
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/DaemonCommon.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon;
+
+public interface DaemonCommon {
+ public boolean isWaiting();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/metrics/MetricsUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/metrics/MetricsUtils.java b/storm-core/src/jvm/org/apache/storm/daemon/metrics/MetricsUtils.java
index 56b920b..eb72939 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/metrics/MetricsUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/metrics/MetricsUtils.java
@@ -20,6 +20,7 @@ package org.apache.storm.daemon.metrics;
import org.apache.storm.Config;
import org.apache.storm.daemon.metrics.reporters.JmxPreparableReporter;
import org.apache.storm.daemon.metrics.reporters.PreparableReporter;
+import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -85,8 +86,8 @@ public class MetricsUtils {
public static File getCsvLogDir(Map stormConf) {
String csvMetricsLogDirectory = Utils.getString(stormConf.get(Config.STORM_DAEMON_METRICS_REPORTER_CSV_LOG_DIR), null);
if (csvMetricsLogDirectory == null) {
- csvMetricsLogDirectory = absoluteStormLocalDir(stormConf);
- csvMetricsLogDirectory = csvMetricsLogDirectory + File.separator + "csvmetrics";
+ csvMetricsLogDirectory = ConfigUtils.absoluteStormLocalDir(stormConf);
+ csvMetricsLogDirectory = csvMetricsLogDirectory + ConfigUtils.FILE_SEPARATOR + "csvmetrics";
}
File csvMetricsDir = new File(csvMetricsLogDirectory);
validateCreateOutputDir(csvMetricsDir);
@@ -104,18 +105,4 @@ public class MetricsUtils {
throw new IllegalStateException(dir.getName() + " is not a directory.");
}
}
-
- public static String absoluteStormLocalDir(Map conf) {
- String stormHome = System.getProperty("storm.home");
- String localDir = (String) conf.get(Config.STORM_LOCAL_DIR);
- if (localDir == null) {
- return (stormHome + File.separator + "storm-local");
- } else {
- if (new File(localDir).isAbsolute()) {
- return localDir;
- } else {
- return (stormHome + File.separator + localDir);
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
new file mode 100644
index 0000000..361328e
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
@@ -0,0 +1,335 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.storm.Config;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AdvancedFSOps {
+ private static final Logger LOG = LoggerFactory.getLogger(AdvancedFSOps.class);
+
+ /**
+ * Factory to create a new AdvancedFSOps
+ * @param conf the configuration of the process
+ * @return the appropriate instance of the class for this config and environment.
+ */
+ public static AdvancedFSOps make(Map<String, Object> conf) {
+ if (Utils.isOnWindows()) {
+ return new AdvancedWindowsFSOps(conf);
+ }
+ if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
+ return new AdvancedRunAsUserFSOps(conf);
+ }
+ return new AdvancedFSOps();
+ }
+
+ private static class AdvancedRunAsUserFSOps extends AdvancedFSOps {
+ private final Map<String, Object> _conf;
+
+ public AdvancedRunAsUserFSOps(Map<String, Object> conf) {
+ if (Utils.isOnWindows()) {
+ throw new UnsupportedOperationException("ERROR: Windows doesn't support running workers as different users yet");
+ }
+ _conf = conf;
+ }
+
+ @Override
+ public void setupBlobPermissions(File path, String user) throws IOException {
+ String logPrefix = "setup blob permissions for " + path;
+ SupervisorUtils.processLauncherAndWait(_conf, user, Arrays.asList("blob", path.toString()), null, logPrefix);
+ }
+
+ @Override
+ public void deleteIfExists(File path, String user, String logPrefix) throws IOException {
+ String absolutePath = path.getAbsolutePath();
+ LOG.info("Deleting path {}", absolutePath);
+ if (user == null) {
+ user = Files.getOwner(path.toPath()).getName();
+ }
+ List<String> commands = new ArrayList<>();
+ commands.add("rmr");
+ commands.add(absolutePath);
+ SupervisorUtils.processLauncherAndWait(_conf, user, commands, null, logPrefix);
+ if (Utils.checkFileExists(absolutePath)) {
+ throw new RuntimeException(path + " was not deleted.");
+ }
+ }
+
+ @Override
+ public void setupStormCodeDir(Map<String, Object> topologyConf, File path) throws IOException {
+ SupervisorUtils.setupStormCodeDir(_conf, topologyConf, path.getCanonicalPath());
+ }
+ }
+
+ /**
+ * Operations that need to override the default ones when running on Windows
+ *
+ */
+ private static class AdvancedWindowsFSOps extends AdvancedFSOps {
+
+ public AdvancedWindowsFSOps(Map<String, Object> conf) {
+ if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
+ throw new RuntimeException("ERROR: Windows doesn't support running workers as different users yet");
+ }
+ }
+
+ @Override
+ public void restrictDirectoryPermissions(File dir) throws IOException {
+ //NOOP, if windows gets support for run as user we will need to find a way to support this
+ }
+
+ @Override
+ public void moveDirectoryPreferAtomic(File fromDir, File toDir) throws IOException {
+ // Files/move with non-empty directory doesn't work well on Windows
+ // This is not atomic but it does work
+ FileUtils.moveDirectory(fromDir, toDir);
+ }
+
+ @Override
+ public boolean supportsAtomicDirectoryMove() {
+ // Files/move with non-empty directory doesn't work well on Windows
+ // FileUtils.moveDirectory is not atomic
+ return false;
+ }
+ }
+
+
+ protected AdvancedFSOps() {
+ //NOOP, but restricted permissions
+ }
+
+ /**
+ * Set directory permissions to (OWNER)RWX (GROUP)R-X (OTHER)---
+ * On some systems that do not support this, it may become a noop
+ * @param dir the directory to change permissions on
+ * @throws IOException on any error
+ */
+ public void restrictDirectoryPermissions(File dir) throws IOException {
+ Set<PosixFilePermission> perms = new HashSet<>(
+ Arrays.asList(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE,
+ PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_READ,
+ PosixFilePermission.GROUP_EXECUTE));
+ Files.setPosixFilePermissions(dir.toPath(), perms);
+ }
+
+ /**
+ * Move fromDir to toDir, and try to make it an atomic move if possible
+ * @param fromDir what to move
+ * @param toDir where to move it from
+ * @throws IOException on any error
+ */
+ public void moveDirectoryPreferAtomic(File fromDir, File toDir) throws IOException {
+ FileUtils.forceMkdir(toDir);
+ Files.move(fromDir.toPath(), toDir.toPath(), StandardCopyOption.ATOMIC_MOVE);
+ }
+
+ /**
+ * @return true if an atomic directory move works, else false.
+ */
+ public boolean supportsAtomicDirectoryMove() {
+ return true;
+ }
+
+ /**
+ * Copy a directory
+ * @param fromDir from where
+ * @param toDir to where
+ * @throws IOException on any error
+ */
+ public void copyDirectory(File fromDir, File toDir) throws IOException {
+ FileUtils.copyDirectory(fromDir, toDir);
+ }
+
+ /**
+ * Setup permissions properly for an internal blob store path
+ * @param path the path to set the permissions on
+ * @param user the user to change the permissions for
+ * @throws IOException on any error
+ */
+ public void setupBlobPermissions(File path, String user) throws IOException {
+ //Normally this is a NOOP
+ }
+
+ /**
+ * Delete a file or a directory and all of the children. If it exists.
+ * @param path what to delete
+ * @param user who to delete it as if doing it as someone else is supported
+ * @param logPrefix if an external process needs to be launched to delete
+ * the object what prefix to include in the logs
+ * @throws IOException on any error.
+ */
+ public void deleteIfExists(File path, String user, String logPrefix) throws IOException {
+ //by default no need to do this as a different user
+ deleteIfExists(path);
+ }
+
+ /**
+ * Delete a file or a directory and all of the children. If it exists.
+ * @param path what to delete
+ * @throws IOException on any error.
+ */
+ public void deleteIfExists(File path) throws IOException {
+ LOG.info("Deleting path {}", path);
+ Path p = path.toPath();
+ if (Files.exists(p)) {
+ try {
+ FileUtils.forceDelete(path);
+ } catch (FileNotFoundException ignored) {}
+ }
+ }
+
+ /**
+ * Setup the permissions for the storm code dir
+ * @param topologyConf the config of the Topology
+ * @param path the directory to set the permissions on
+ * @throws IOException on any error
+ */
+ public void setupStormCodeDir(Map<String, Object> topologyConf, File path) throws IOException {
+ //By default this is a NOOP
+ }
+
+ /**
+ * Sanity check if everything the topology needs is there for it to run.
+ * @param conf the config of the supervisor
+ * @param topologyId the ID of the topology
+ * @return true if everything is there, else false
+ * @throws IOException on any error
+ */
+ public boolean doRequiredTopoFilesExist(Map<String, Object> conf, String topologyId) throws IOException {
+ return SupervisorUtils.doRequiredTopoFilesExist(conf, topologyId);
+ }
+
+ /**
+ * Makes a directory, including any necessary but nonexistent parent
+ * directories.
+ *
+ * @param path the directory to create
+ * @throws IOException on any error
+ */
+ public void forceMkdir(File path) throws IOException {
+ FileUtils.forceMkdir(path);
+ }
+
+ /**
+ * Check if a file exists or not
+ * @param path the path to check
+ * @return true if it exists else false
+ * @throws IOException on any error.
+ */
+ public boolean fileExists(File path) throws IOException {
+ return path.exists();
+ }
+
+ /**
+ * Get a writer for the given location
+ * @param file the file to write to
+ * @return the Writer to use.
+ * @throws IOException on any error
+ */
+ public Writer getWriter(File file) throws IOException {
+ return new FileWriter(file);
+ }
+
+ /**
+ * Get an output stream to write to a given file
+ * @param file the file to write to
+ * @return an OutputStream for that file
+ * @throws IOException on any error
+ */
+ public OutputStream getOutputStream(File file) throws IOException {
+ return new FileOutputStream(file);
+ }
+
+ /**
+ * Dump a string to a file
+ * @param location where to write to
+ * @param data the data to write
+ * @throws IOException on any error
+ */
+ public void dump(File location, String data) throws IOException {
+ File parent = location.getParentFile();
+ if (!parent.exists()) {
+ forceMkdir(parent);
+ }
+ try (Writer w = getWriter(location)) {
+ w.write(data);
+ }
+ }
+
+ /**
+ * Read the contents of a file into a String
+ * @param location the file to read
+ * @return the contents of the file
+ * @throws IOException on any error
+ */
+ public String slurpString(File location) throws IOException {
+ return FileUtils.readFileToString(location, "UTF-8");
+ }
+
+ /**
+ * Read the contents of a file into a byte array.
+ * @param localtion the file to read
+ * @return the contents of the file
+ * @throws IOException on any error
+ */
+ public byte[] slurp(File location) throws IOException {
+ return FileUtils.readFileToByteArray(location);
+ }
+
+ /**
+ * Create a symbolic link pointing at target
+ * @param link the link to create
+ * @param target where it should point to
+ * @throws IOException on any error.
+ */
+ public void createSymlink(File link, File target) throws IOException {
+ Path plink = link.toPath().toAbsolutePath();
+ Path ptarget = target.toPath().toAbsolutePath();
+ LOG.debug("Creating symlink [{}] to [{}]", plink, ptarget);
+ if (Files.exists(plink)) {
+ if (Files.isSameFile(plink, ptarget)) {
+ //It already points where we want it to
+ return;
+ }
+ FileUtils.forceDelete(link);
+ }
+ Files.createSymbolicLink(plink, ptarget);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
new file mode 100644
index 0000000..237d89b
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -0,0 +1,638 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.Config;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.generated.WorkerResources;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+
+/**
+ * A container that runs processes on the local box.
+ */
+public class BasicContainer extends Container {
+ private static final Logger LOG = LoggerFactory.getLogger(BasicContainer.class);
+ private static final FilenameFilter jarFilter = (dir, name) -> name.endsWith(".jar");
+ private static final Joiner CPJ =
+ Joiner.on(Utils.CLASS_PATH_SEPARATOR).skipNulls();
+
+ protected final LocalState _localState;
+ protected final String _profileCmd;
+ protected final String _stormHome = System.getProperty("storm.home");
+ protected volatile boolean _exitedEarly = false;
+
+ private class ProcessExitCallback implements ExitCodeCallback {
+ private final String _logPrefix;
+
+ public ProcessExitCallback(String logPrefix) {
+ _logPrefix = logPrefix;
+ }
+
+ @Override
+ public void call(int exitCode) {
+ LOG.info("{} exited with code: {}", _logPrefix, exitCode);
+ _exitedEarly = true;
+ }
+ }
+
+ /**
+ * Create a new BasicContainer
+ * @param type the type of container being made.
+ * @param conf the supervisor config
+ * @param supervisorId the ID of the supervisor this is a part of.
+ * @param port the port the container is on. Should be <= 0 if only a partial recovery
+ * @param assignment the assignment for this container. Should be null if only a partial recovery.
+ * @param localState the local state of the supervisor. May be null if partial recovery
+ * @param workerId the id of the worker to use. Must not be null if doing a partial recovery.
+ */
+ public BasicContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+ LocalAssignment assignment,
+ LocalState localState, String workerId) throws IOException {
+ this(type, conf, supervisorId, port, assignment, localState, workerId, null, null, null);
+ }
+
+ /**
+ * Create a new BasicContainer
+ * @param type the type of container being made.
+ * @param conf the supervisor config
+ * @param supervisorId the ID of the supervisor this is a part of.
+ * @param port the port the container is on. Should be <= 0 if only a partial recovery
+ * @param assignment the assignment for this container. Should be null if only a partial recovery.
+ * @param localState the local state of the supervisor. May be null if partial recovery
+ * @param workerId the id of the worker to use. Must not be null if doing a partial recovery.
+ * @param ops file system operations (mostly for testing) if null a new one is made
+ * @param topoConf the config of the topology (mostly for testing) if null
+ * and not a partial recovery the real conf is read.
+ * @param profileCmd the command to use when profiling (used for testing)
+ * @throws IOException on any error
+ * @throws ContainerRecoveryException if the Container could not be recovered.
+ */
+ BasicContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+ LocalAssignment assignment,
+ LocalState localState, String workerId, Map<String, Object> topoConf,
+ AdvancedFSOps ops, String profileCmd) throws IOException {
+ super(type, conf, supervisorId, port, assignment, workerId, topoConf, ops);
+ assert(localState != null);
+ _localState = localState;
+
+ if (type.isRecovery() && !type.isOnlyKillable()) {
+ synchronized (localState) {
+ String wid = null;
+ Map<String, Integer> workerToPort = localState.getApprovedWorkers();
+ for (Map.Entry<String, Integer> entry : workerToPort.entrySet()) {
+ if (port == entry.getValue().intValue()) {
+ wid = entry.getKey();
+ }
+ }
+ if (wid == null) {
+ throw new ContainerRecoveryException("Could not find worker id for " + port + " " + assignment);
+ }
+ LOG.info("Recovered Worker {}", wid);
+ _workerId = wid;
+ }
+ } else if (_workerId == null){
+ createNewWorkerId();
+ }
+
+ if (profileCmd == null) {
+ profileCmd = _stormHome + Utils.FILE_PATH_SEPARATOR + "bin" + Utils.FILE_PATH_SEPARATOR
+ + conf.get(Config.WORKER_PROFILER_COMMAND);
+ }
+ _profileCmd = profileCmd;
+ }
+
+ /**
+ * Create a new worker ID for this process and store in in this object and
+ * in the local state. Never call this if a worker is currently up and running.
+ * We will lose track of the process.
+ */
+ protected void createNewWorkerId() {
+ _type.assertFull();
+ assert(_workerId == null);
+ synchronized (_localState) {
+ _workerId = Utils.uuid();
+ Map<String, Integer> workerToPort = _localState.getApprovedWorkers();
+ if (workerToPort == null) {
+ workerToPort = new HashMap<>(1);
+ }
+ removeWorkersOn(workerToPort, _port);
+ workerToPort.put(_workerId, _port);
+ _localState.setApprovedWorkers(workerToPort);
+ LOG.info("Created Worker ID {}", _workerId);
+ }
+ }
+
+ private static void removeWorkersOn(Map<String, Integer> workerToPort, int _port) {
+ for (Iterator<Entry<String, Integer>> i = workerToPort.entrySet().iterator(); i.hasNext();) {
+ Entry<String, Integer> found = i.next();
+ if (_port == found.getValue().intValue()) {
+ LOG.warn("Deleting worker {} from state", found.getKey());
+ i.remove();
+ }
+ }
+ }
+
+ @Override
+ public void cleanUpForRestart() throws IOException {
+ String origWorkerId = _workerId;
+ super.cleanUpForRestart();
+ synchronized (_localState) {
+ Map<String, Integer> workersToPort = _localState.getApprovedWorkers();
+ workersToPort.remove(origWorkerId);
+ removeWorkersOn(workersToPort, _port);
+ _localState.setApprovedWorkers(workersToPort);
+ LOG.info("Removed Worker ID {}", origWorkerId);
+ }
+ }
+
+ @Override
+ public void relaunch() throws IOException {
+ _type.assertFull();
+ //We are launching it now...
+ _type = ContainerType.LAUNCH;
+ createNewWorkerId();
+ setup();
+ launch();
+ }
+
+ @Override
+ public boolean didMainProcessExit() {
+ return _exitedEarly;
+ }
+
+ /**
+ * Run the given command for profiling
+ *
+ * @param command
+ * the command to run
+ * @param env
+ * the environment to run the command
+ * @param logPrefix
+ * the prefix to include in the logs
+ * @param targetDir
+ * the working directory to run the command in
+ * @return true if it ran successfully, else false
+ * @throws IOException
+ * on any error
+ * @throws InterruptedException
+ * if interrupted wile waiting for the process to exit.
+ */
+ protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix,
+ File targetDir) throws IOException, InterruptedException {
+ _type.assertFull();
+ Process p = SupervisorUtils.launchProcess(command, env, logPrefix, null, targetDir);
+ int ret = p.waitFor();
+ return ret == 0;
+ }
+
+ @Override
+ public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException {
+ _type.assertFull();
+ String targetDir = ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port);
+
+ @SuppressWarnings("unchecked")
+ Map<String, String> env = (Map<String, String>) _topoConf.get(Config.TOPOLOGY_ENVIRONMENT);
+ if (env == null) {
+ env = new HashMap<String, String>();
+ }
+
+ String str = ConfigUtils.workerArtifactsPidPath(_conf, _topologyId, _port);
+
+ String workerPid = _ops.slurpString(new File(str)).trim();
+
+ ProfileAction profileAction = request.get_action();
+ String logPrefix = "ProfilerAction process " + _topologyId + ":" + _port + " PROFILER_ACTION: " + profileAction
+ + " ";
+
+ List<String> command = mkProfileCommand(profileAction, stop, workerPid, targetDir);
+
+ File targetFile = new File(targetDir);
+ if (command.size() > 0) {
+ return runProfilingCommand(command, env, logPrefix, targetFile);
+ }
+ LOG.warn("PROFILING REQUEST NOT SUPPORTED {} IGNORED...", request);
+ return true;
+ }
+
+ /**
+ * Get the command to run when doing profiling
+ * @param action the profiling action to perform
+ * @param stop if this is meant to stop the profiling or start it
+ * @param workerPid the PID of the process to profile
+ * @param targetDir the current working directory of the worker process
+ * @return the command to run for profiling.
+ */
+ private List<String> mkProfileCommand(ProfileAction action, boolean stop, String workerPid, String targetDir) {
+ switch(action) {
+ case JMAP_DUMP:
+ return jmapDumpCmd(workerPid, targetDir);
+ case JSTACK_DUMP:
+ return jstackDumpCmd(workerPid, targetDir);
+ case JPROFILE_DUMP:
+ return jprofileDump(workerPid, targetDir);
+ case JVM_RESTART:
+ return jprofileJvmRestart(workerPid);
+ case JPROFILE_STOP:
+ if (stop) {
+ return jprofileStop(workerPid, targetDir);
+ }
+ return jprofileStart(workerPid);
+ default:
+ return Lists.newArrayList();
+ }
+ }
+
+ private List<String> jmapDumpCmd(String pid, String targetDir) {
+ return Lists.newArrayList(_profileCmd, pid, "jmap", targetDir);
+ }
+
+ private List<String> jstackDumpCmd(String pid, String targetDir) {
+ return Lists.newArrayList(_profileCmd, pid, "jstack", targetDir);
+ }
+
+ private List<String> jprofileStart(String pid) {
+ return Lists.newArrayList(_profileCmd, pid, "start");
+ }
+
+ private List<String> jprofileStop(String pid, String targetDir) {
+ return Lists.newArrayList(_profileCmd, pid, "stop", targetDir);
+ }
+
+ private List<String> jprofileDump(String pid, String targetDir) {
+ return Lists.newArrayList(_profileCmd, pid, "dump", targetDir);
+ }
+
+ private List<String> jprofileJvmRestart(String pid) {
+ return Lists.newArrayList(_profileCmd, pid, "kill");
+ }
+
+ /**
+ * Compute the java.library.path that should be used for the worker.
+ * This helps it to load JNI libraries that are packaged in the uber jar.
+ * @param stormRoot the root directory of the worker process
+ * @param conf the config for the supervisor.
+ * @return the java.library.path/LD_LIBRARY_PATH to use so native libraries load correctly.
+ */
+ protected String javaLibraryPath(String stormRoot, Map<String, Object> conf) {
+ String resourceRoot = stormRoot + Utils.FILE_PATH_SEPARATOR + ConfigUtils.RESOURCES_SUBDIR;
+ String os = System.getProperty("os.name").replaceAll("\\s+", "_");
+ String arch = System.getProperty("os.arch");
+ String archResourceRoot = resourceRoot + Utils.FILE_PATH_SEPARATOR + os + "-" + arch;
+ String ret = CPJ.join(archResourceRoot, resourceRoot,
+ conf.get(Config.JAVA_LIBRARY_PATH));
+ return ret;
+ }
+
+ /**
+ * Returns a collection of jar file names found under the given directory.
+ * @param dir the directory to search
+ * @return the jar file names
+ */
+ protected List<String> getFullJars(File dir) {
+ File[] files = dir.listFiles(jarFilter);
+
+ if (files == null) {
+ return Collections.emptyList();
+ }
+
+ return Arrays.stream(files).map(f -> f.getAbsolutePath())
+ .collect(Collectors.toList());
+ }
+
+ protected List<String> frameworkClasspath() {
+ File stormLibDir = new File(_stormHome, "lib");
+ String stormConfDir =
+ System.getenv("STORM_CONF_DIR") != null ?
+ System.getenv("STORM_CONF_DIR") :
+ new File(_stormHome, "conf").getAbsolutePath();
+ File stormExtlibDir = new File(_stormHome, "extlib");
+ String extcp = System.getenv("STORM_EXT_CLASSPATH");
+ List<String> pathElements = new LinkedList<>();
+ pathElements.addAll(getFullJars(stormLibDir));
+ pathElements.addAll(getFullJars(stormExtlibDir));
+ pathElements.add(extcp);
+ pathElements.add(stormConfDir);
+
+ return pathElements;
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<String> asStringList(Object o) {
+ if (o instanceof String) {
+ return Arrays.asList((String)o);
+ } else if (o instanceof List) {
+ return (List<String>)o;
+ }
+ return Collections.EMPTY_LIST;
+ }
+
+ /**
+ * Compute the classpath for the worker process
+ * @param stormJar the topology jar
+ * @param dependencyLocations any dependencies from the topology
+ * @return the full classpath
+ */
+ protected String getWorkerClassPath(String stormJar, List<String> dependencyLocations) {
+ List<String> workercp = new ArrayList<>();
+ workercp.addAll(frameworkClasspath());
+ workercp.add(stormJar);
+ workercp.addAll(dependencyLocations);
+ workercp.addAll(asStringList(_topoConf.get(Config.TOPOLOGY_CLASSPATH)));
+ return CPJ.join(workercp);
+ }
+
+ private String substituteChildOptsInternal(String string, int memOnheap) {
+ if (StringUtils.isNotBlank(string)) {
+ String p = String.valueOf(_port);
+ string = string.replace("%ID%", p);
+ string = string.replace("%WORKER-ID%", _workerId);
+ string = string.replace("%TOPOLOGY-ID%", _topologyId);
+ string = string.replace("%WORKER-PORT%", p);
+ if (memOnheap > 0) {
+ string = string.replace("%HEAP-MEM%", String.valueOf(memOnheap));
+ }
+ }
+ return string;
+ }
+
+ protected List<String> substituteChildopts(Object value) {
+ return substituteChildopts(value, -1);
+ }
+
+ protected List<String> substituteChildopts(Object value, int memOnheap) {
+ List<String> rets = new ArrayList<>();
+ if (value instanceof String) {
+ String string = substituteChildOptsInternal((String) value, memOnheap);
+ if (StringUtils.isNotBlank(string)) {
+ String[] strings = string.split("\\s+");
+ rets.addAll(Arrays.asList(strings));
+ }
+ } else if (value instanceof List) {
+ @SuppressWarnings("unchecked")
+ List<String> objects = (List<String>) value;
+ for (String object : objects) {
+ String str = substituteChildOptsInternal(object, memOnheap);
+ if (StringUtils.isNotBlank(str)) {
+ rets.add(str);
+ }
+ }
+ }
+ return rets;
+ }
+
+ /**
+ * Launch the worker process (non-blocking)
+ *
+ * @param command
+ * the command to run
+ * @param env
+ * the environment to run the command
+ * @param processExitcallback
+ * a callback for when the process exits
+ * @param logPrefix
+ * the prefix to include in the logs
+ * @param targetDir
+ * the working directory to run the command in
+ * @return true if it ran successfully, else false
+ * @throws IOException
+ * on any error
+ */
+ protected void launchWorkerProcess(List<String> command, Map<String, String> env, String logPrefix,
+ ExitCodeCallback processExitCallback, File targetDir) throws IOException {
+ SupervisorUtils.launchProcess(command, env, logPrefix, processExitCallback, targetDir);
+ }
+
+ private String getWorkerLoggingConfigFile() {
+ String log4jConfigurationDir = (String) (_conf.get(Config.STORM_LOG4J2_CONF_DIR));
+
+ if (StringUtils.isNotBlank(log4jConfigurationDir)) {
+ if (!Utils.isAbsolutePath(log4jConfigurationDir)) {
+ log4jConfigurationDir = _stormHome + Utils.FILE_PATH_SEPARATOR + log4jConfigurationDir;
+ }
+ } else {
+ log4jConfigurationDir = _stormHome + Utils.FILE_PATH_SEPARATOR + "log4j2";
+ }
+
+ if (Utils.IS_ON_WINDOWS && !log4jConfigurationDir.startsWith("file:")) {
+ log4jConfigurationDir = "file:///" + log4jConfigurationDir;
+ }
+ return log4jConfigurationDir + Utils.FILE_PATH_SEPARATOR + "worker.xml";
+ }
+
+ /**
+ * Get parameters for the class path of the worker process. Also used by the
+ * log Writer
+ * @param stormRoot the root dist dir for the topology
+ * @return the classpath for the topology as command line arguments.
+ * @throws IOException on any error.
+ */
+ private List<String> getClassPathParams(final String stormRoot) throws IOException {
+ final String stormJar = ConfigUtils.supervisorStormJarPath(stormRoot);
+ final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _ops);
+ final List<String> dependencyLocations = new ArrayList<>();
+ if (stormTopology.get_dependency_jars() != null) {
+ for (String dependency : stormTopology.get_dependency_jars()) {
+ dependencyLocations.add(new File(stormRoot, dependency).getAbsolutePath());
+ }
+ }
+
+ if (stormTopology.get_dependency_artifacts() != null) {
+ for (String dependency : stormTopology.get_dependency_artifacts()) {
+ dependencyLocations.add(new File(stormRoot, dependency).getAbsolutePath());
+ }
+ }
+ final String workerClassPath = getWorkerClassPath(stormJar, dependencyLocations);
+
+ List<String> classPathParams = new ArrayList<>();
+ classPathParams.add("-cp");
+ classPathParams.add(workerClassPath);
+ return classPathParams;
+ }
+
+ /**
+ * Get a set of java properties that are common to both the log writer and the worker processes.
+ * These are mostly system properties that are used by logging.
+ * @return a list of command line options
+ */
+ private List<String> getCommonParams() {
+ final String workersArtifacts = ConfigUtils.workerArtifactsRoot(_conf);
+ String stormLogDir = ConfigUtils.getLogDir();
+ String log4jConfigurationFile = getWorkerLoggingConfigFile();
+
+ List<String> commonParams = new ArrayList<>();
+ commonParams.add("-Dlogging.sensitivity=" + OR((String) _topoConf.get(Config.TOPOLOGY_LOGGING_SENSITIVITY), "S3"));
+ commonParams.add("-Dlogfile.name=worker.log");
+ commonParams.add("-Dstorm.home=" + OR(_stormHome, ""));
+ commonParams.add("-Dworkers.artifacts=" + workersArtifacts);
+ commonParams.add("-Dstorm.id=" + _topologyId);
+ commonParams.add("-Dworker.id=" + _workerId);
+ commonParams.add("-Dworker.port=" + _port);
+ commonParams.add("-Dstorm.log.dir=" + stormLogDir);
+ commonParams.add("-Dlog4j.configurationFile=" + log4jConfigurationFile);
+ commonParams.add("-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector");
+ commonParams.add("-Dstorm.local.dir=" + _conf.get(Config.STORM_LOCAL_DIR));
+ return commonParams;
+ }
+
+ private int getMemOnHeap(WorkerResources resources) {
+ int memOnheap = 0;
+ if (resources != null && resources.is_set_mem_on_heap() &&
+ resources.get_mem_on_heap() > 0) {
+ memOnheap = (int) Math.ceil(resources.get_mem_on_heap());
+ } else {
+ // set the default heap memory size for supervisor-test
+ memOnheap = Utils.getInt(_topoConf.get(Config.WORKER_HEAP_MEMORY_MB), 768);
+ }
+ return memOnheap;
+ }
+
+ private List<String> getWorkerProfilerChildOpts(int memOnheap) {
+ List<String> workerProfilerChildopts = new ArrayList<>();
+ if (Utils.getBoolean(_conf.get(Config.WORKER_PROFILER_ENABLED), false)) {
+ workerProfilerChildopts = substituteChildopts(_conf.get(Config.WORKER_PROFILER_CHILDOPTS), memOnheap);
+ }
+ return workerProfilerChildopts;
+ }
+
+ /**
+ * a or b the first one that is not null
+ * @param a something
+ * @param b something else
+ * @return a or b the first one that is not null
+ */
+ private <V> V OR(V a, V b) {
+ return a == null ? b : a;
+ }
+
+ protected String javaCmd(String cmd) {
+ String ret = null;
+ String javaHome = System.getenv().get("JAVA_HOME");
+ if (StringUtils.isNotBlank(javaHome)) {
+ ret = javaHome + Utils.FILE_PATH_SEPARATOR + "bin" + Utils.FILE_PATH_SEPARATOR + cmd;
+ } else {
+ ret = cmd;
+ }
+ return ret;
+ }
+
+ /**
+ * Create the command to launch the worker process
+ * @param memOnheap the on heap memory for the worker
+ * @param stormRoot the root dist dir for the topology
+ * @param jlp java library path for the topology
+ * @return the command to run
+ * @throws IOException on any error.
+ */
+ private List<String> mkLaunchCommand(final int memOnheap, final String stormRoot,
+ final String jlp) throws IOException {
+ final String javaCmd = javaCmd("java");
+ final String stormOptions = ConfigUtils.concatIfNotNull(System.getProperty("storm.options"));
+ final String stormConfFile = ConfigUtils.concatIfNotNull(System.getProperty("storm.conf.file"));
+ final String workerTmpDir = ConfigUtils.workerTmpRoot(_conf, _workerId);
+
+ List<String> classPathParams = getClassPathParams(stormRoot);
+ List<String> commonParams = getCommonParams();
+
+ List<String> commandList = new ArrayList<>();
+ //Log Writer Command...
+ commandList.add(javaCmd);
+ commandList.addAll(classPathParams);
+ commandList.addAll(substituteChildopts(_topoConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS)));
+ commandList.addAll(commonParams);
+ commandList.add("org.apache.storm.LogWriter"); //The LogWriter in turn launches the actual worker.
+
+ //Worker Command...
+ commandList.add(javaCmd);
+ commandList.add("-server");
+ commandList.addAll(commonParams);
+ commandList.addAll(substituteChildopts(_conf.get(Config.WORKER_CHILDOPTS), memOnheap));
+ commandList.addAll(substituteChildopts(_topoConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), memOnheap));
+ commandList.addAll(substituteChildopts(OR(
+ _topoConf.get(Config.TOPOLOGY_WORKER_GC_CHILDOPTS),
+ _conf.get(Config.WORKER_GC_CHILDOPTS)), memOnheap));
+ commandList.addAll(getWorkerProfilerChildOpts(memOnheap));
+ commandList.add("-Djava.library.path=" + jlp);
+ commandList.add("-Dstorm.conf.file=" + stormConfFile);
+ commandList.add("-Dstorm.options=" + stormOptions);
+ commandList.add("-Djava.io.tmpdir=" + workerTmpDir);
+ commandList.addAll(classPathParams);
+ commandList.add("org.apache.storm.daemon.worker");
+ commandList.add(_topologyId);
+ commandList.add(_supervisorId);
+ commandList.add(String.valueOf(_port));
+ commandList.add(_workerId);
+
+ return commandList;
+ }
+
+ @Override
+ public void launch() throws IOException {
+ _type.assertFull();
+ LOG.info("Launching worker with assignment {} for this supervisor {} on port {} with id {}", _assignment,
+ _supervisorId, _port, _workerId);
+ String logPrefix = "Worker Process " + _workerId;
+ ProcessExitCallback processExitCallback = new ProcessExitCallback(logPrefix);
+ _exitedEarly = false;
+
+ final WorkerResources resources = _assignment.get_resources();
+ final int memOnheap = getMemOnHeap(resources);
+ final String stormRoot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId);
+ final String jlp = javaLibraryPath(stormRoot, _conf);
+
+ List<String> commandList = mkLaunchCommand(memOnheap, stormRoot, jlp);
+
+ Map<String, String> topEnvironment = new HashMap<String, String>();
+ @SuppressWarnings("unchecked")
+ Map<String, String> environment = (Map<String, String>) _topoConf.get(Config.TOPOLOGY_ENVIRONMENT);
+ if (environment != null) {
+ topEnvironment.putAll(environment);
+ }
+ topEnvironment.put("LD_LIBRARY_PATH", jlp);
+
+ LOG.info("Launching worker with command: {}. ", Utils.shellCmd(commandList));
+
+ String workerDir = ConfigUtils.workerRoot(_conf, _workerId);
+
+ launchWorkerProcess(commandList, topEnvironment, logPrefix, processExitCallback, new File(workerDir));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java
new file mode 100644
index 0000000..7dacf14
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.storm.daemon.supervisor.Container.ContainerType;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.utils.LocalState;
+
+/**
+ * Launch containers with no security using standard java commands
+ */
+public class BasicContainerLauncher extends ContainerLauncher {
+ private final Map<String, Object> _conf;
+ private final String _supervisorId;
+
+ public BasicContainerLauncher(Map<String, Object> conf, String supervisorId) throws IOException {
+ _conf = conf;
+ _supervisorId = supervisorId;
+ }
+
+ @Override
+ public Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+ Container container = new BasicContainer(ContainerType.LAUNCH, _conf, _supervisorId, port, assignment,
+ state, null);
+ container.setup();
+ container.launch();
+ return container;
+ }
+
+ @Override
+ public Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+ return new BasicContainer(ContainerType.RECOVER_FULL, _conf, _supervisorId, port, assignment,
+ state, null);
+ }
+
+ @Override
+ public Killable recoverContainer(String workerId, LocalState localState) throws IOException {
+ return new BasicContainer(ContainerType.RECOVER_PARTIAL, _conf, _supervisorId, -1, null,
+ localState, workerId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
new file mode 100644
index 0000000..aea4b53
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
@@ -0,0 +1,534 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Writer;
+import java.lang.ProcessBuilder.Redirect;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.storm.Config;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+
+/**
+ * Represents a container that a worker will run in.
+ */
+public abstract class Container implements Killable {
+ private static final Logger LOG = LoggerFactory.getLogger(Container.class);
+ public static enum ContainerType {
+ LAUNCH(false, false),
+ RECOVER_FULL(true, false),
+ RECOVER_PARTIAL(true, true);
+
+ private final boolean _recovery;
+ private final boolean _onlyKillable;
+
+ ContainerType(boolean recovery, boolean onlyKillable) {
+ _recovery = recovery;
+ _onlyKillable = onlyKillable;
+ }
+
+ public boolean isRecovery() {
+ return _recovery;
+ }
+
+ public void assertFull() {
+ if (_onlyKillable) {
+ throw new IllegalStateException("Container is only Killable.");
+ }
+ }
+
+ public boolean isOnlyKillable() {
+ return _onlyKillable;
+ }
+ }
+
+ protected final Map<String, Object> _conf;
+ protected final Map<String, Object> _topoConf; //Not set if RECOVER_PARTIAL
+ protected String _workerId;
+ protected final String _topologyId; //Not set if RECOVER_PARTIAL
+ protected final String _supervisorId;
+ protected final int _port; //Not set if RECOVER_PARTIAL
+ protected final LocalAssignment _assignment; //Not set if RECOVER_PARTIAL
+ protected final AdvancedFSOps _ops;
+ protected ContainerType _type;
+
+ /**
+ * Create a new Container.
+ * @param type the type of container being made.
+ * @param conf the supervisor config
+ * @param supervisorId the ID of the supervisor this is a part of.
+ * @param port the port the container is on. Should be <= 0 if only a partial recovery
+ * @param assignment the assignment for this container. Should be null if only a partial recovery.
+ * @param workerId the id of the worker to use. Must not be null if doing a partial recovery.
+ * @param topoConf the config of the topology (mostly for testing) if null
+ * and not a partial recovery the real conf is read.
+ * @param ops file system operations (mostly for testing) if null a new one is made
+ * @throws IOException on any error.
+ */
+ protected Container(ContainerType type, Map<String, Object> conf, String supervisorId,
+ int port, LocalAssignment assignment,
+ String workerId, Map<String, Object> topoConf, AdvancedFSOps ops) throws IOException {
+ assert(type != null);
+ assert(conf != null);
+ assert(supervisorId != null);
+
+ if (ops == null) {
+ ops = AdvancedFSOps.make(conf);
+ }
+
+ _workerId = workerId;
+ _type = type;
+ _port = port;
+ _ops = ops;
+ _conf = conf;
+ _supervisorId = supervisorId;
+ _assignment = assignment;
+
+ if (_type.isOnlyKillable()) {
+ assert(_assignment == null);
+ assert(_port <= 0);
+ assert(_workerId != null);
+ _topologyId = null;
+ _topoConf = null;
+ } else {
+ assert(assignment != null);
+ assert(port > 0);
+ _topologyId = assignment.get_topology_id();
+ if (!_ops.doRequiredTopoFilesExist(_conf, _topologyId)) {
+ LOG.info("Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}", _assignment,
+ _supervisorId, _port, _workerId);
+ throw new ContainerRecoveryException("Missing required topology files...");
+ }
+ if (topoConf == null) {
+ _topoConf = readTopoConf();
+ } else {
+ //For testing...
+ _topoConf = topoConf;
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "topo:" + _topologyId + " worker:" + _workerId;
+ }
+
+ protected Map<String, Object> readTopoConf() throws IOException {
+ assert(_topologyId != null);
+ return ConfigUtils.readSupervisorStormConf(_conf, _topologyId);
+ }
+
+ /**
+ * Kill a given process
+ * @param pid the id of the process to kill
+ * @throws IOException
+ */
+ protected void kill(long pid) throws IOException {
+ Utils.killProcessWithSigTerm(String.valueOf(pid));
+ }
+
+ /**
+ * Kill a given process
+ * @param pid the id of the process to kill
+ * @throws IOException
+ */
+ protected void forceKill(long pid) throws IOException {
+ Utils.forceKillProcess(String.valueOf(pid));
+ }
+
+ @Override
+ public void kill() throws IOException {
+ LOG.info("Killing {}:{}", _supervisorId, _workerId);
+ Set<Long> pids = getAllPids();
+
+ for (Long pid : pids) {
+ kill(pid);
+ }
+ }
+
+ @Override
+ public void forceKill() throws IOException {
+ LOG.info("Force Killing {}:{}", _supervisorId, _workerId);
+ Set<Long> pids = getAllPids();
+
+ for (Long pid : pids) {
+ forceKill(pid);
+ }
+ }
+
+ /**
+ * Read the Heartbeat for the current container.
+ * @return the Heartbeat
+ * @throws IOException on any error
+ */
+ public LSWorkerHeartbeat readHeartbeat() throws IOException {
+ LocalState localState = ConfigUtils.workerState(_conf, _workerId);
+ LSWorkerHeartbeat hb = localState.getWorkerHeartBeat();
+ LOG.trace("{}: Reading heartbeat {}", _workerId, hb);
+ return hb;
+ }
+
+ /**
+ * Is a process alive and running?
+ * @param pid the PID of the running process
+ * @param user the user that is expected to own that process
+ * @return true if it is, else false
+ * @throws IOException on any error
+ */
+ protected boolean isProcessAlive(long pid, String user) throws IOException {
+ if (Utils.IS_ON_WINDOWS) {
+ return isWindowsProcessAlive(pid, user);
+ }
+ return isPosixProcessAlive(pid, user);
+ }
+
+ private boolean isWindowsProcessAlive(long pid, String user) throws IOException {
+ boolean ret = false;
+ ProcessBuilder pb = new ProcessBuilder("tasklist", "/fo", "list", "/fi", "pid eq " + pid, "/v");
+ pb.redirectError(Redirect.INHERIT);
+ Process p = pb.start();
+ try (BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
+ String read;
+ while ((read = in.readLine()) != null) {
+ if (read.contains("User Name:")) { //Check for : in case someone called their user "User Name"
+ //This line contains the user name for the pid we're looking up
+ //Example line: "User Name: exampleDomain\exampleUser"
+ List<String> userNameLineSplitOnWhitespace = Arrays.asList(read.split(":"));
+ if(userNameLineSplitOnWhitespace.size() == 2){
+ List<String> userAndMaybeDomain = Arrays.asList(userNameLineSplitOnWhitespace.get(1).trim().split("\\\\"));
+ String processUser = userAndMaybeDomain.size() == 2 ? userAndMaybeDomain.get(1) : userAndMaybeDomain.get(0);
+ if(user.equals(processUser)){
+ ret = true;
+ } else {
+ LOG.info("Found {} running as {}, but expected it to be {}", pid, processUser, user);
+ }
+ } else {
+ LOG.error("Received unexpected output from tasklist command. Expected one colon in user name line. Line was {}", read);
+ }
+ break;
+ }
+ }
+ }
+ return ret;
+ }
+
+ private boolean isPosixProcessAlive(long pid, String user) throws IOException {
+ boolean ret = false;
+ ProcessBuilder pb = new ProcessBuilder("ps", "-o", "user", "-p", String.valueOf(pid));
+ pb.redirectError(Redirect.INHERIT);
+ Process p = pb.start();
+ try (BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
+ String first = in.readLine();
+ assert("USER".equals(first));
+ String processUser;
+ while ((processUser = in.readLine()) != null) {
+ if (user.equals(processUser)) {
+ ret = true;
+ break;
+ } else {
+ LOG.info("Found {} running as {}, but expected it to be {}", pid, processUser, user);
+ }
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public boolean areAllProcessesDead() throws IOException {
+ Set<Long> pids = getAllPids();
+ String user = getWorkerUser();
+
+ boolean allDead = true;
+ for (Long pid: pids) {
+ if (!isProcessAlive(pid, user)) {
+ LOG.debug("{}: PID {} is dead", _workerId, pid);
+ } else {
+ allDead = false;
+ break;
+ }
+ }
+ return allDead;
+ }
+
+ @Override
+ public void cleanUp() throws IOException {
+ cleanUpForRestart();
+ }
+
+ /**
+ * Setup the container to run. By default this creates the needed directories/links in the
+ * local file system
+ * PREREQUISITE: All needed blobs and topology, jars/configs have been downloaded and
+ * placed in the appropriate locations
+ * @throws IOException on any error
+ */
+ protected void setup() throws IOException {
+ _type.assertFull();
+ if (!_ops.doRequiredTopoFilesExist(_conf, _topologyId)) {
+ LOG.info("Missing topology storm code, so can't launch worker with assignment {} for this supervisor {} on port {} with id {}", _assignment,
+ _supervisorId, _port, _workerId);
+ throw new IllegalStateException("Not all needed files are here!!!!");
+ }
+ LOG.info("Setting up {}:{}", _supervisorId, _workerId);
+
+ _ops.forceMkdir(new File(ConfigUtils.workerPidsRoot(_conf, _workerId)));
+ _ops.forceMkdir(new File(ConfigUtils.workerTmpRoot(_conf, _workerId)));
+ _ops.forceMkdir(new File(ConfigUtils.workerHeartbeatsRoot(_conf, _workerId)));
+
+ File workerArtifacts = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port));
+ if (!_ops.fileExists(workerArtifacts)) {
+ _ops.forceMkdir(workerArtifacts);
+ _ops.setupStormCodeDir(_topoConf, workerArtifacts);
+ }
+
+ String user = getWorkerUser();
+ writeLogMetadata(user);
+ saveWorkerUser(user);
+ createArtifactsLink();
+ createBlobstoreLinks();
+ }
+
+ /**
+ * Write out the file used by the log viewer to allow/reject log access
+ * @param user the user this is going to run as
+ * @throws IOException on any error
+ */
+ @SuppressWarnings("unchecked")
+ protected void writeLogMetadata(String user) throws IOException {
+ _type.assertFull();
+ Map<String, Object> data = new HashMap<>();
+ data.put(Config.TOPOLOGY_SUBMITTER_USER, user);
+ data.put("worker-id", _workerId);
+
+ Set<String> logsGroups = new HashSet<>();
+ if (_topoConf.get(Config.LOGS_GROUPS) != null) {
+ List<String> groups = (List<String>) _topoConf.get(Config.LOGS_GROUPS);
+ for (String group : groups){
+ logsGroups.add(group);
+ }
+ }
+ if (_topoConf.get(Config.TOPOLOGY_GROUPS) != null) {
+ List<String> topGroups = (List<String>) _topoConf.get(Config.TOPOLOGY_GROUPS);
+ logsGroups.addAll(topGroups);
+ }
+ data.put(Config.LOGS_GROUPS, logsGroups.toArray());
+
+ Set<String> logsUsers = new HashSet<>();
+ if (_topoConf.get(Config.LOGS_USERS) != null) {
+ List<String> logUsers = (List<String>) _topoConf.get(Config.LOGS_USERS);
+ for (String logUser : logUsers){
+ logsUsers.add(logUser);
+ }
+ }
+ if (_topoConf.get(Config.TOPOLOGY_USERS) != null) {
+ List<String> topUsers = (List<String>) _topoConf.get(Config.TOPOLOGY_USERS);
+ for (String logUser : topUsers){
+ logsUsers.add(logUser);
+ }
+ }
+ data.put(Config.LOGS_USERS, logsUsers.toArray());
+
+ File file = ConfigUtils.getLogMetaDataFile(_conf, _topologyId, _port);
+
+ Yaml yaml = new Yaml();
+ try (Writer writer = _ops.getWriter(file)) {
+ yaml.dump(data, writer);
+ }
+ }
+
+ /**
+ * Create symlink from the containers directory/artifacts to the artifacts directory
+ * @throws IOException on any error
+ */
+ protected void createArtifactsLink() throws IOException {
+ _type.assertFull();
+ File workerDir = new File(ConfigUtils.workerRoot(_conf, _workerId));
+ File topoDir = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port));
+ if (_ops.fileExists(workerDir)) {
+ LOG.debug("Creating symlinks for worker-id: {} topology-id: {} to its port artifacts directory", _workerId, _topologyId);
+ _ops.createSymlink(new File(workerDir, "artifacts"), topoDir);
+ }
+ }
+
+ /**
+ * Create symlinks for each of the blobs from the container's directory to
+ * corresponding links in the storm dist directory.
+ * @throws IOException on any error.
+ */
+ protected void createBlobstoreLinks() throws IOException {
+ _type.assertFull();
+ String stormRoot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId);
+ String workerRoot = ConfigUtils.workerRoot(_conf, _workerId);
+
+ @SuppressWarnings("unchecked")
+ Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) _topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
+ List<String> blobFileNames = new ArrayList<>();
+ if (blobstoreMap != null) {
+ for (Map.Entry<String, Map<String, Object>> entry : blobstoreMap.entrySet()) {
+ String key = entry.getKey();
+ Map<String, Object> blobInfo = entry.getValue();
+ String ret = null;
+ if (blobInfo != null && blobInfo.containsKey("localname")) {
+ ret = (String) blobInfo.get("localname");
+ } else {
+ ret = key;
+ }
+ blobFileNames.add(ret);
+ }
+ }
+ List<String> resourceFileNames = new ArrayList<>();
+ resourceFileNames.add(ConfigUtils.RESOURCES_SUBDIR);
+ resourceFileNames.addAll(blobFileNames);
+ LOG.info("Creating symlinks for worker-id: {} storm-id: {} for files({}): {}", _workerId, _topologyId, resourceFileNames.size(), resourceFileNames);
+ _ops.createSymlink(new File(workerRoot, ConfigUtils.RESOURCES_SUBDIR),
+ new File(stormRoot, ConfigUtils.RESOURCES_SUBDIR));
+ for (String fileName : blobFileNames) {
+ _ops.createSymlink(new File(workerRoot, fileName),
+ new File(stormRoot, fileName));
+ }
+ }
+
+ /**
+ * @return all of the pids that are a part of this container.
+ */
+ protected Set<Long> getAllPids() throws IOException {
+ Set<Long> ret = new HashSet<>();
+ for (String listing: Utils.readDirContents(ConfigUtils.workerPidsRoot(_conf, _workerId))) {
+ ret.add(Long.valueOf(listing));
+ }
+
+ return ret;
+ }
+
+ /**
+ * @return the user that some operations should be done as.
+ * @throws IOException on any error
+ */
+ protected String getWorkerUser() throws IOException {
+ LOG.info("GET worker-user for {}", _workerId);
+ File file = new File(ConfigUtils.workerUserFile(_conf, _workerId));
+
+ if (_ops.fileExists(file)) {
+ return _ops.slurpString(file).trim();
+ } else if (_topoConf != null) {
+ return (String) _topoConf.get(Config.TOPOLOGY_SUBMITTER_USER);
+ }
+ if (ConfigUtils.isLocalMode(_conf)) {
+ return System.getProperty("user.name");
+ } else {
+ File f = new File(ConfigUtils.workerArtifactsRoot(_conf));
+ if (f.exists()) {
+ return Files.getOwner(f.toPath()).getName();
+ }
+ throw new IllegalStateException("Could not recover the user for " + _workerId);
+ }
+ }
+
+ protected void saveWorkerUser(String user) throws IOException {
+ _type.assertFull();
+ LOG.info("SET worker-user {} {}", _workerId, user);
+ _ops.dump(new File(ConfigUtils.workerUserFile(_conf, _workerId)), user);
+ }
+
+ protected void deleteSavedWorkerUser() throws IOException {
+ LOG.info("REMOVE worker-user {}", _workerId);
+ _ops.deleteIfExists(new File(ConfigUtils.workerUserFile(_conf, _workerId)));
+ }
+
+ /**
+ * Clean up the container partly preparing for restart.
+ * By default delete all of the temp directories we are going
+ * to get a new worker_id anyways.
+ * POST CONDITION: the workerId will be set to null
+ * @throws IOException on any error
+ */
+ public void cleanUpForRestart() throws IOException {
+ LOG.info("Cleaning up {}:{}", _supervisorId, _workerId);
+ Set<Long> pids = getAllPids();
+ String user = getWorkerUser();
+
+ for (Long pid : pids) {
+ File path = new File(ConfigUtils.workerPidPath(_conf, _workerId, pid));
+ _ops.deleteIfExists(path, user, _workerId);
+ }
+
+ //Always make sure to clean up everything else before worker directory
+ //is removed since that is what is going to trigger the retry for cleanup
+ _ops.deleteIfExists(new File(ConfigUtils.workerHeartbeatsRoot(_conf, _workerId)), user, _workerId);
+ _ops.deleteIfExists(new File(ConfigUtils.workerPidsRoot(_conf, _workerId)), user, _workerId);
+ _ops.deleteIfExists(new File(ConfigUtils.workerTmpRoot(_conf, _workerId)), user, _workerId);
+ _ops.deleteIfExists(new File(ConfigUtils.workerRoot(_conf, _workerId)), user, _workerId);
+ deleteSavedWorkerUser();
+ _workerId = null;
+ }
+
+ /**
+ * Launch the process for the first time
+ * PREREQUISITE: setup has run and passed
+ * @throws IOException on any error
+ */
+ public abstract void launch() throws IOException;
+
+ /**
+ * Restart the processes in this container
+ * PREREQUISITE: cleanUpForRestart has run and passed
+ * @throws IOException on any error
+ */
+ public abstract void relaunch() throws IOException;
+
+ /**
+ * @return true if the main process exited, else false. This is just best effort return false if unknown.
+ */
+ public abstract boolean didMainProcessExit();
+
+ /**
+ * Run a profiling request
+ * @param request the request to run
+ * @param stop is this a stop request?
+ * @return true if it succeeded, else false
+ * @throws IOException on any error
+ * @throws InterruptedException if running the command is interrupted.
+ */
+ public abstract boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException;
+
+ /**
+ * @return the id of the container or null if there is no worker id right now.
+ */
+ public String getWorkerId() {
+ return _workerId;
+ }
+}
[02/21] storm git commit: STORM-2018: Supervisor V2
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/integration/org/apache/storm/integration_test.clj b/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
index 238e0db..11e22fb 100644
--- a/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
+++ b/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
@@ -20,10 +20,14 @@
(:import [org.apache.storm.generated InvalidTopologyException SubmitOptions TopologyInitialStatus RebalanceOptions])
(:import [org.apache.storm.testing TestWordCounter TestWordSpout TestGlobalCount
TestAggregatesCounter TestConfBolt AckFailMapTracker AckTracker TestPlannerSpout])
+ (:import [org.apache.storm.task WorkerTopologyContext])
(:import [org.apache.storm.utils Time])
(:import [org.apache.storm.tuple Fields])
+ (:import [org.mockito Mockito])
(:use [org.apache.storm testing config clojure util])
(:use [org.apache.storm.daemon common])
+ (:require [org.apache.storm [cluster :as cluster]])
+ (:require [org.apache.storm.daemon [executor :as executor]])
(:require [org.apache.storm [thrift :as thrift]]))
(deftest test-basic-topology
@@ -115,7 +119,6 @@
"timeout-tester"
{TOPOLOGY-MESSAGE-TIMEOUT-SECS 10}
topology)
- (advance-cluster-time cluster 11)
(.feed feeder ["a"] 1)
(.feed feeder ["b"] 2)
(.feed feeder ["c"] 3)
@@ -282,7 +285,6 @@
"acking-test1"
{}
(:topology tracked))
- (advance-cluster-time cluster 11)
(.feed feeder1 [1])
(tracked-wait tracked 1)
(checker1 0)
@@ -325,7 +327,6 @@
"test-acking2"
{}
(:topology tracked))
- (advance-cluster-time cluster 11)
(.feed feeder [1])
(tracked-wait tracked 1)
(checker 0)
@@ -371,8 +372,7 @@
{TOPOLOGY-MESSAGE-TIMEOUT-SECS 10}
topology
(SubmitOptions. TopologyInitialStatus/INACTIVE))
- (advance-cluster-time cluster 11)
- (.feed feeder ["a"] 1)
+ (.feedNoWait feeder ["a"] 1)
(advance-cluster-time cluster 9)
(is (not @bolt-prepared?))
(is (not @spout-opened?))
@@ -396,7 +396,6 @@
"test"
{}
(:topology tracked))
- (advance-cluster-time cluster 11)
(.feed feeder [1])
(tracked-wait tracked 1)
(checker 1)
@@ -595,60 +594,42 @@
(read-tuples results "2")
)))))
-(defbolt report-errors-bolt {}
- [tuple collector]
- (doseq [i (range (.getValue tuple 0))]
- (report-error! collector (RuntimeException.)))
- (ack! collector tuple))
-
+;;This is more of a unit test, but getting the timing right for
+;; an integration test is really hard
(deftest test-throttled-errors
(with-simulated-time
- (with-tracked-cluster [cluster]
- (let [state (:storm-cluster-state cluster)
- [feeder checker] (ack-tracking-feeder ["num"])
- tracked (mk-tracked-topology
- cluster
- (topology
- {"1" (spout-spec feeder)}
- {"2" (bolt-spec {"1" :shuffle} report-errors-bolt)}))
- _ (submit-local-topology (:nimbus cluster)
- "test-errors"
- {TOPOLOGY-ERROR-THROTTLE-INTERVAL-SECS 10
- TOPOLOGY-MAX-ERROR-REPORT-PER-INTERVAL 4
- TOPOLOGY-DEBUG true
- }
- (:topology tracked))
- _ (advance-cluster-time cluster 11)
- storm-id (get-storm-id state "test-errors")
- errors-count (fn [] (count (.errors state storm-id "2")))]
-
- (is (nil? (.last-error state storm-id "2")))
-
- ;; so it launches the topology
- (advance-cluster-time cluster 2)
- (.feed feeder [6])
- (tracked-wait tracked 1)
- (is (= 4 (errors-count)))
- (is (.last-error state storm-id "2"))
-
- (advance-time-secs! 5)
- (.feed feeder [2])
- (tracked-wait tracked 1)
- (is (= 4 (errors-count)))
- (is (.last-error state storm-id "2"))
-
- (advance-time-secs! 6)
- (.feed feeder [2])
- (tracked-wait tracked 1)
- (is (= 6 (errors-count)))
- (is (.last-error state storm-id "2"))
-
- (advance-time-secs! 6)
- (.feed feeder [3])
- (tracked-wait tracked 1)
- (is (= 8 (errors-count)))
- (is (.last-error state storm-id "2"))))))
-
+ (let [error-count (atom 0)
+ worker-context (Mockito/mock WorkerTopologyContext)
+ cluster-state
+ (reify cluster/StormClusterState
+ (report-error
+ [this storm-id component-id node port error]
+ (swap! error-count inc)))
+ error-fn (executor/throttled-report-error-fn
+ {:storm-conf {TOPOLOGY-ERROR-THROTTLE-INTERVAL-SECS 10
+ TOPOLOGY-MAX-ERROR-REPORT-PER-INTERVAL 4}
+ :storm-cluster-state cluster-state
+ :storm-id "topo"
+ :component-id "comp"
+ :worker-context worker-context})]
+ (. (Mockito/when (.getThisWorkerPort worker-context)) (thenReturn (Integer. 8080)))
+ (error-fn (RuntimeException. "ERROR-1"))
+ (is (= 1 @error-count))
+ (error-fn (RuntimeException. "ERROR-2"))
+ (is (= 2 @error-count))
+ (error-fn (RuntimeException. "ERROR-3"))
+ (is (= 3 @error-count))
+ (error-fn (RuntimeException. "ERROR-4"))
+ (is (= 4 @error-count))
+ ;;Ignored
+ (error-fn (RuntimeException. "ERROR-5"))
+ (is (= 4 @error-count))
+ (Time/advanceTime 9000)
+ (error-fn (RuntimeException. "ERROR-6"))
+ (is (= 4 @error-count))
+ (Time/advanceTime 2000)
+ (error-fn (RuntimeException. "ERROR-7"))
+ (is (= 5 @error-count)))))
(deftest test-acking-branching-complex
;; test acking with branching in the topology
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj b/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
index 5f71e86..f3955bb 100644
--- a/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
+++ b/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
@@ -121,9 +121,9 @@
(it/agg-bolt 4))}))]
(.submitTopology cluster
"test-acking2"
- (Config.)
+ (doto (Config.)
+ (.put TOPOLOGY-DEBUG true))
(.getTopology tracked))
- (advance-cluster-time (.getState cluster) 11)
(.feed feeder [1])
(Testing/trackedWait tracked (int 1))
(checker 0)
@@ -153,13 +153,14 @@
"timeout-tester"
storm-conf
topology)
+ ;;Wait for the topology to come up
(.feed feeder ["a"] 1)
(.feed feeder ["b"] 2)
(.feed feeder ["c"] 3)
(Testing/advanceClusterTime cluster (int 9))
(it/assert-acked tracker 1 3)
(is (not (.isFailed tracker 2)))
- (Testing/advanceClusterTime cluster (int 12))
+ (Testing/advanceClusterTime cluster (int 20))
(it/assert-failed tracker 2)
))))))
@@ -179,19 +180,21 @@
{"1" (thrift/mk-spout-spec feeder)}
{"2" (thrift/mk-bolt-spec {"1" :global} it/ack-every-other)})
storm-conf (doto (Config.)
+ (.put TOPOLOGY-DEBUG true)
(.put TOPOLOGY-MESSAGE-TIMEOUT-SECS 10)
(.put TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS false))]
(.submitTopology cluster
"disable-timeout-tester"
storm-conf
topology)
+ ;;Wait for the topology to come up
(.feed feeder ["a"] 1)
(.feed feeder ["b"] 2)
(.feed feeder ["c"] 3)
(Testing/advanceClusterTime cluster (int 9))
(it/assert-acked tracker 1 3)
(is (not (.isFailed tracker 2)))
- (Testing/advanceClusterTime cluster (int 12))
+ (Testing/advanceClusterTime cluster (int 20))
(is (not (.isFailed tracker 2)))
))))))
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/org/apache/storm/logviewer_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/logviewer_test.clj b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
index 4d32a06..aae76e6 100644
--- a/storm-core/test/clj/org/apache/storm/logviewer_test.clj
+++ b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
@@ -15,13 +15,16 @@
;; limitations under the License.
(ns org.apache.storm.logviewer-test
(:use [org.apache.storm config util])
- (:require [org.apache.storm.daemon [logviewer :as logviewer]
- [supervisor :as supervisor]])
+ (:require [org.apache.storm.daemon [logviewer :as logviewer]])
(:require [conjure.core])
(:use [clojure test])
(:use [conjure core])
- (:use [org.apache.storm.ui helpers])
- (:import [org.apache.storm.daemon DirectoryCleaner])
+ (:use [org.apache.storm testing]
+ [org.apache.storm.ui helpers])
+ (:import [org.apache.storm.daemon DirectoryCleaner]
+ [org.apache.storm.daemon.supervisor SupervisorUtils]
+ [org.apache.storm.testing.staticmocking MockedSupervisorUtils]
+ [org.apache.storm.generated LSWorkerHeartbeat])
(:import [java.nio.file Files Path DirectoryStream])
(:import [java.nio.file Files])
(:import [java.nio.file.attribute FileAttribute])
@@ -231,27 +234,33 @@
mock-metaFile (mk-mock-File {:name "worker.yaml"
:type :file})
exp-id "id12345"
- expected {exp-id port1-dir}]
- (stubbing [supervisor/read-worker-heartbeats nil
- logviewer/get-metadata-file-for-wroker-logdir mock-metaFile
- logviewer/get-worker-id-from-metadata-file exp-id]
- (is (= expected (logviewer/identify-worker-log-dirs [port1-dir])))))))
+ expected {exp-id port1-dir}
+ supervisor-util (Mockito/mock SupervisorUtils)]
+ (with-open [_ (MockedSupervisorUtils. supervisor-util)]
+ (stubbing [logviewer/get-metadata-file-for-wroker-logdir mock-metaFile
+ logviewer/get-worker-id-from-metadata-file exp-id]
+ (. (Mockito/when (.readWorkerHeartbeatsImpl supervisor-util (Mockito/any))) (thenReturn nil))
+ (is (= expected (logviewer/identify-worker-log-dirs [port1-dir]))))))))
(deftest test-get-dead-worker-dirs
- (testing "return directories for workers that are not alive"
- (let [conf {SUPERVISOR-WORKER-TIMEOUT-SECS 5}
- id->hb {"42" {:time-secs 1}} ;; map for alive ids
- now-secs 2
- unexpected-dir1 (mk-mock-File {:name "dir1" :type :directory})
- expected-dir2 (mk-mock-File {:name "dir2" :type :directory})
- expected-dir3 (mk-mock-File {:name "dir3" :type :directory})
- log-dirs #{unexpected-dir1 expected-dir2 expected-dir3}]
- (stubbing [logviewer/identify-worker-log-dirs {"42" unexpected-dir1,
- "007" expected-dir2,
- "" expected-dir3} ;; this tests a directory with no yaml file thus no worker id
- supervisor/read-worker-heartbeats id->hb]
- (is (= #{expected-dir2 expected-dir3}
- (logviewer/get-dead-worker-dirs conf now-secs log-dirs)))))))
+ (testing "return directories for workers that are not alive"
+ (let [conf {SUPERVISOR-WORKER-TIMEOUT-SECS 5}
+ hb (let [lwb (LSWorkerHeartbeat.)]
+ (.set_time_secs lwb (int 1)) lwb)
+ id->hb {"42" hb}
+ now-secs 2
+ unexpected-dir1 (mk-mock-File {:name "dir1" :type :directory})
+ expected-dir2 (mk-mock-File {:name "dir2" :type :directory})
+ expected-dir3 (mk-mock-File {:name "dir3" :type :directory})
+ log-dirs #{unexpected-dir1 expected-dir2 expected-dir3}
+ supervisor-util (Mockito/mock SupervisorUtils)]
+ (with-open [_ (MockedSupervisorUtils. supervisor-util)]
+ (stubbing [logviewer/identify-worker-log-dirs {"42" unexpected-dir1,
+ "007" expected-dir2,
+ "" expected-dir3}] ;; this tests a directory with no yaml file thus no worker id
+ (. (Mockito/when (.readWorkerHeartbeatsImpl supervisor-util (Mockito/any))) (thenReturn id->hb))
+ (is (= #{expected-dir2 expected-dir3}
+ (logviewer/get-dead-worker-dirs conf now-secs log-dirs))))))))
(deftest test-cleanup-fn
(testing "cleanup function rmr's files of dead workers"
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/org/apache/storm/metrics_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/metrics_test.clj b/storm-core/test/clj/org/apache/storm/metrics_test.clj
index 9f051f6..ba48ea9 100644
--- a/storm-core/test/clj/org/apache/storm/metrics_test.clj
+++ b/storm-core/test/clj/org/apache/storm/metrics_test.clj
@@ -260,18 +260,18 @@
{"mybolt" (thrift/mk-bolt-spec {"myspout" :shuffle} ack-every-other)})]
(submit-local-topology (:nimbus cluster)
"metrics-tester"
- {}
+ {TOPOLOGY-DEBUG true}
topology)
(.feed feeder ["a"] 1)
(advance-cluster-time cluster 6)
- (assert-acked tracker 1)
(assert-buckets! "myspout" "__fail-count/default" [] cluster)
(assert-buckets! "myspout" "__ack-count/default" [1] cluster)
(assert-buckets! "myspout" "__emit-count/default" [1] cluster)
(assert-buckets! "myspout" "__transfer-count/default" [1] cluster)
(assert-buckets! "mybolt" "__ack-count/myspout:default" [1] cluster)
(assert-buckets! "mybolt" "__execute-count/myspout:default" [1] cluster)
+ (assert-acked tracker 1)
(.feed feeder ["b"] 2)
(advance-cluster-time cluster 5)
@@ -312,18 +312,19 @@
{"mybolt" (thrift/mk-bolt-spec {"myspout" :global} ack-every-other)})]
(submit-local-topology (:nimbus cluster)
"timeout-tester"
- {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10}
+ {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10
+ TOPOLOGY-DEBUG true}
topology)
(.feed feeder ["a"] 1)
(.feed feeder ["b"] 2)
(.feed feeder ["c"] 3)
(advance-cluster-time cluster 9)
- (assert-acked tracker 1 3)
(assert-buckets! "myspout" "__ack-count/default" [2] cluster)
(assert-buckets! "myspout" "__emit-count/default" [3] cluster)
(assert-buckets! "myspout" "__transfer-count/default" [3] cluster)
(assert-buckets! "mybolt" "__ack-count/myspout:default" [2] cluster)
(assert-buckets! "mybolt" "__execute-count/myspout:default" [3] cluster)
+ (assert-acked tracker 1 3)
(is (not (.isFailed tracker 2)))
(advance-cluster-time cluster 30)
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/org/apache/storm/nimbus_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/nimbus_test.clj b/storm-core/test/clj/org/apache/storm/nimbus_test.clj
index 7a1724d..c7b00b3 100644
--- a/storm-core/test/clj/org/apache/storm/nimbus_test.clj
+++ b/storm-core/test/clj/org/apache/storm/nimbus_test.clj
@@ -1056,6 +1056,14 @@
(assert-files-in-dir [])
))))
+(defn wait-for-status [nimbus name status]
+ (while-timeout 5000
+ (let [topo-summary (first (filter (fn [topo] (= name (.get_name topo))) (.get_topologies (.getClusterInfo nimbus))))
+ topo-status (if topo-summary (.get_status topo-summary) "NOT-RUNNING")]
+ (log-message "WAITING FOR "name" TO BE " status " CURRENT " topo-status)
+ (not= topo-status status))
+ (Thread/sleep 100)))
+
(deftest test-leadership
"Tests that leader actions can only be performed by master and non leader fails to perform the same actions."
(with-inprocess-zookeeper zk-port
@@ -1082,7 +1090,7 @@
(submit-local-topology nimbus "t1" {} topology)
;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
(.rebalance nimbus "t1" (doto (RebalanceOptions.) (.set_wait_secs 0)))
- (Thread/sleep 1000)
+ (wait-for-status nimbus "t1" "ACTIVE")
(.deactivate nimbus "t1")
(.activate nimbus "t1")
(.rebalance nimbus "t1" (RebalanceOptions.))
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj b/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
index d6a431f..c1eacd6 100644
--- a/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
+++ b/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
@@ -98,42 +98,42 @@
(defmacro with-simple-drpc-test-scenario
[[strict? alice-client bob-client charlie-client alice-invok charlie-invok] & body]
- (let [client-port (available-port)
- invocations-port (available-port (inc client-port))
- storm-conf (merge (read-storm-config)
- {DRPC-AUTHORIZER-ACL-STRICT strict?
+ `(let [client-port# (available-port)
+ invocations-port# (available-port (inc client-port#))
+ storm-conf# (merge (read-storm-config)
+ {DRPC-AUTHORIZER-ACL-STRICT ~strict?
DRPC-AUTHORIZER-ACL-FILENAME "drpc-simple-acl-test-scenario.yaml"
STORM-THRIFT-TRANSPORT-PLUGIN "org.apache.storm.security.auth.digest.DigestSaslTransportPlugin"})]
- `(with-server [~storm-conf
+ (with-server [storm-conf#
"org.apache.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer"
"org.apache.storm.security.auth.digest.DigestSaslTransportPlugin"
"test/clj/org/apache/storm/security/auth/drpc-auth-server.jaas"
- ~client-port ~invocations-port]
+ client-port# invocations-port#]
(let [~alice-client (DRPCClient.
- (merge ~storm-conf {"java.security.auth.login.config"
+ (merge storm-conf# {"java.security.auth.login.config"
"test/clj/org/apache/storm/security/auth/drpc-auth-alice.jaas"})
"localhost"
- ~client-port)
+ client-port#)
~bob-client (DRPCClient.
- (merge ~storm-conf {"java.security.auth.login.config"
+ (merge storm-conf# {"java.security.auth.login.config"
"test/clj/org/apache/storm/security/auth/drpc-auth-bob.jaas"})
"localhost"
- ~client-port)
+ client-port#)
~charlie-client (DRPCClient.
- (merge ~storm-conf {"java.security.auth.login.config"
+ (merge storm-conf# {"java.security.auth.login.config"
"test/clj/org/apache/storm/security/auth/drpc-auth-charlie.jaas"})
"localhost"
- ~client-port)
+ client-port#)
~alice-invok (DRPCInvocationsClient.
- (merge ~storm-conf {"java.security.auth.login.config"
+ (merge storm-conf# {"java.security.auth.login.config"
"test/clj/org/apache/storm/security/auth/drpc-auth-alice.jaas"})
"localhost"
- ~invocations-port)
+ invocations-port#)
~charlie-invok (DRPCInvocationsClient.
- (merge ~storm-conf {"java.security.auth.login.config"
+ (merge storm-conf# {"java.security.auth.login.config"
"test/clj/org/apache/storm/security/auth/drpc-auth-charlie.jaas"})
"localhost"
- ~invocations-port)]
+ invocations-port#)]
(try
~@body
(finally
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/org/apache/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/supervisor_test.clj b/storm-core/test/clj/org/apache/storm/supervisor_test.clj
deleted file mode 100644
index 997de83..0000000
--- a/storm-core/test/clj/org/apache/storm/supervisor_test.clj
+++ /dev/null
@@ -1,751 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.supervisor-test
- (:use [clojure test])
- (:require [conjure.core])
- (:use [conjure core])
- (:require [clojure.contrib [string :as contrib-str]])
- (:require [clojure [string :as string] [set :as set]])
- (:import [org.apache.storm.testing TestWordCounter TestWordSpout TestGlobalCount TestAggregatesCounter TestPlannerSpout])
- (:import [org.apache.storm.scheduler ISupervisor])
- (:import [org.apache.storm.generated RebalanceOptions])
- (:import [java.util UUID])
- (:import [java.io File])
- (:import [java.nio.file Files])
- (:import [java.nio.file.attribute FileAttribute] (org.apache.storm.topology TopologyBuilder))
- (:use [org.apache.storm config testing util timer])
- (:use [org.apache.storm.daemon common])
- (:require [org.apache.storm.daemon [worker :as worker] [supervisor :as supervisor]]
- [org.apache.storm [thrift :as thrift] [cluster :as cluster]])
- (:use [conjure core])
- (:require [clojure.java.io :as io]))
-
-(defn worker-assignment
- "Return [storm-id executors]"
- [cluster supervisor-id port]
- (let [state (:storm-cluster-state cluster)
- slot-assigns (for [storm-id (.assignments state nil)]
- (let [executors (-> (.assignment-info state storm-id nil)
- :executor->node+port
- reverse-map
- (get [supervisor-id port] ))]
- (when executors [storm-id executors])
- ))
- ret (find-first not-nil? slot-assigns)]
- (when-not ret
- (throw-runtime "Could not find assignment for worker"))
- ret
- ))
-
-(defn heartbeat-worker [supervisor port storm-id executors]
- (let [conf (.get-conf supervisor)]
- (worker/do-heartbeat {:conf conf
- :port port
- :storm-id storm-id
- :executors executors
- :worker-id (find-worker-id conf port)})))
-
-(defn heartbeat-workers [cluster supervisor-id ports]
- (let [sup (get-supervisor cluster supervisor-id)]
- (doseq [p ports]
- (let [[storm-id executors] (worker-assignment cluster supervisor-id p)]
- (heartbeat-worker sup p storm-id executors)
- ))))
-
-(defn validate-launched-once [launched supervisor->ports storm-id]
- (let [counts (map count (vals launched))
- launched-supervisor->ports (apply merge-with set/union
- (for [[[s p] sids] launched
- :when (some #(= % storm-id) sids)]
- {s #{p}}))
- supervisor->ports (map-val set supervisor->ports)]
- (is (every? (partial = 1) counts))
- (is (= launched-supervisor->ports supervisor->ports))
- ))
-
-(deftest launches-assignment
- (with-simulated-time-local-cluster [cluster :supervisors 0
- :daemon-conf {NIMBUS-DO-NOT-REASSIGN true
- SUPERVISOR-WORKER-START-TIMEOUT-SECS 5
- SUPERVISOR-WORKER-TIMEOUT-SECS 15
- SUPERVISOR-MONITOR-FREQUENCY-SECS 3}]
- (letlocals
- (bind topology (thrift/mk-topology
- {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 4)}
- {}))
- (bind sup1 (add-supervisor cluster :id "sup1" :ports [1 2 3 4]))
- (bind changed (capture-changed-workers
- (submit-mocked-assignment
- (:nimbus cluster)
- (:storm-cluster-state cluster)
- "test"
- {TOPOLOGY-WORKERS 3}
- topology
- {1 "1"
- 2 "1"
- 3 "1"
- 4 "1"}
- {[1 1] ["sup1" 1]
- [2 2] ["sup1" 2]
- [3 3] ["sup1" 3]
- [4 4] ["sup1" 3]}
- {["sup1" 1] [0.0 0.0 0.0]
- ["sup1" 2] [0.0 0.0 0.0]
- ["sup1" 3] [0.0 0.0 0.0]
- })
- ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
- (.rebalance (:nimbus cluster) "test" (doto (RebalanceOptions.) (.set_wait_secs 0)))
- (advance-cluster-time cluster 2)
- (heartbeat-workers cluster "sup1" [1 2 3])
- (advance-cluster-time cluster 10)))
- (bind storm-id (get-storm-id (:storm-cluster-state cluster) "test"))
- (is (empty? (:shutdown changed)))
- (validate-launched-once (:launched changed) {"sup1" [1 2 3]} storm-id)
- (bind changed (capture-changed-workers
- (doseq [i (range 10)]
- (heartbeat-workers cluster "sup1" [1 2 3])
- (advance-cluster-time cluster 10))
- ))
- (is (empty? (:shutdown changed)))
- (is (empty? (:launched changed)))
- (bind changed (capture-changed-workers
- (heartbeat-workers cluster "sup1" [1 2])
- (advance-cluster-time cluster 10)
- ))
- (validate-launched-once (:launched changed) {"sup1" [3]} storm-id)
- (is (= {["sup1" 3] 1} (:shutdown changed)))
- )))
-
-(deftest test-multiple-active-storms-multiple-supervisors
- (with-simulated-time-local-cluster [cluster :supervisors 0
- :daemon-conf {NIMBUS-DO-NOT-REASSIGN true
- SUPERVISOR-WORKER-START-TIMEOUT-SECS 5
- SUPERVISOR-WORKER-TIMEOUT-SECS 15
- SUPERVISOR-MONITOR-FREQUENCY-SECS 3}]
- (letlocals
- (bind topology (thrift/mk-topology
- {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 4)}
- {}))
- (bind topology2 (thrift/mk-topology
- {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 3)}
- {}))
- (bind sup1 (add-supervisor cluster :id "sup1" :ports [1 2 3 4]))
- (bind sup2 (add-supervisor cluster :id "sup2" :ports [1 2]))
- (bind changed (capture-changed-workers
- (submit-mocked-assignment
- (:nimbus cluster)
- (:storm-cluster-state cluster)
- "test"
- {TOPOLOGY-WORKERS 3 TOPOLOGY-MESSAGE-TIMEOUT-SECS 40}
- topology
- {1 "1"
- 2 "1"
- 3 "1"
- 4 "1"}
- {[1 1] ["sup1" 1]
- [2 2] ["sup1" 2]
- [3 3] ["sup2" 1]
- [4 4] ["sup2" 1]}
- {["sup1" 1] [0.0 0.0 0.0]
- ["sup1" 2] [0.0 0.0 0.0]
- ["sup2" 1] [0.0 0.0 0.0]
- })
- ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
- (.rebalance (:nimbus cluster) "test" (doto (RebalanceOptions.) (.set_wait_secs 0)))
- (advance-cluster-time cluster 2)
- (heartbeat-workers cluster "sup1" [1 2])
- (heartbeat-workers cluster "sup2" [1])
- ))
- (bind storm-id (get-storm-id (:storm-cluster-state cluster) "test"))
- (is (empty? (:shutdown changed)))
- (validate-launched-once (:launched changed) {"sup1" [1 2] "sup2" [1]} storm-id)
- (bind changed (capture-changed-workers
- (submit-mocked-assignment
- (:nimbus cluster)
- (:storm-cluster-state cluster)
- "test2"
- {TOPOLOGY-WORKERS 2}
- topology2
- {1 "1"
- 2 "1"
- 3 "1"}
- {[1 1] ["sup1" 3]
- [2 2] ["sup1" 3]
- [3 3] ["sup2" 2]}
- {["sup1" 3] [0.0 0.0 0.0]
- ["sup2" 2] [0.0 0.0 0.0]
- })
- ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
- (.rebalance (:nimbus cluster) "test2" (doto (RebalanceOptions.) (.set_wait_secs 0)))
- (advance-cluster-time cluster 2)
- (heartbeat-workers cluster "sup1" [3])
- (heartbeat-workers cluster "sup2" [2])
- ))
- (bind storm-id2 (get-storm-id (:storm-cluster-state cluster) "test2"))
- (is (empty? (:shutdown changed)))
- (validate-launched-once (:launched changed) {"sup1" [3] "sup2" [2]} storm-id2)
- (bind changed (capture-changed-workers
- (.killTopology (:nimbus cluster) "test")
- (doseq [i (range 4)]
- (advance-cluster-time cluster 8)
- (heartbeat-workers cluster "sup1" [1 2 3])
- (heartbeat-workers cluster "sup2" [1 2])
- )))
- (is (empty? (:shutdown changed)))
- (is (empty? (:launched changed)))
- (bind changed (capture-changed-workers
- (advance-cluster-time cluster 12)
- ))
- (is (empty? (:launched changed)))
- (is (= {["sup1" 1] 1 ["sup1" 2] 1 ["sup2" 1] 1} (:shutdown changed)))
- (bind changed (capture-changed-workers
- (doseq [i (range 10)]
- (heartbeat-workers cluster "sup1" [3])
- (heartbeat-workers cluster "sup2" [2])
- (advance-cluster-time cluster 10)
- )))
- (is (empty? (:shutdown changed)))
- (is (empty? (:launched changed)))
- ;; TODO check that downloaded code is cleaned up only for the one storm
- )))
-
-(defn get-heartbeat [cluster supervisor-id]
- (.supervisor-info (:storm-cluster-state cluster) supervisor-id))
-
-(defn check-heartbeat [cluster supervisor-id within-secs]
- (let [hb (get-heartbeat cluster supervisor-id)
- time-secs (:time-secs hb)
- now (current-time-secs)
- delta (- now time-secs)]
- (is (>= delta 0))
- (is (<= delta within-secs))
- ))
-
-(deftest heartbeats-to-nimbus
- (with-simulated-time-local-cluster [cluster :supervisors 0
- :daemon-conf {SUPERVISOR-WORKER-START-TIMEOUT-SECS 15
- SUPERVISOR-HEARTBEAT-FREQUENCY-SECS 3}]
- (letlocals
- (bind sup1 (add-supervisor cluster :id "sup" :ports [5 6 7]))
- (advance-cluster-time cluster 4)
- (bind hb (get-heartbeat cluster "sup"))
- (is (= #{5 6 7} (set (:meta hb))))
- (check-heartbeat cluster "sup" 3)
- (advance-cluster-time cluster 3)
- (check-heartbeat cluster "sup" 3)
- (advance-cluster-time cluster 3)
- (check-heartbeat cluster "sup" 3)
- (advance-cluster-time cluster 15)
- (check-heartbeat cluster "sup" 3)
- (bind topology (thrift/mk-topology
- {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 4)}
- {}))
- ;; prevent them from launching by capturing them
- (capture-changed-workers
- (submit-local-topology (:nimbus cluster) "test" {TOPOLOGY-WORKERS 2} topology)
- (advance-cluster-time cluster 3)
- (check-heartbeat cluster "sup" 3)
- (advance-cluster-time cluster 3)
- (check-heartbeat cluster "sup" 3)
- (advance-cluster-time cluster 3)
- (check-heartbeat cluster "sup" 3)
- (advance-cluster-time cluster 20)
- (check-heartbeat cluster "sup" 3))
- )))
-
-(deftest test-worker-launch-command
- (testing "*.worker.childopts configuration"
- (let [mock-port "42"
- mock-storm-id "fake-storm-id"
- mock-worker-id "fake-worker-id"
- mock-mem-onheap 512
- mock-cp (str file-path-separator "base" class-path-separator file-path-separator "stormjar.jar")
- mock-sensitivity "S3"
- mock-cp "/base:/stormjar.jar"
- builder (TopologyBuilder.)
- \u2028_ (.setSpout builder "wordSpout" (TestWordSpout.) 1)
- \u2028_ (.shuffleGrouping (.setBolt builder "wordCountBolt" (TestWordCounter.) 1) "wordSpout")
- \u2028mock-storm-topology (.createTopology builder)
- exp-args-fn (fn [opts topo-opts classpath]
- (concat [(supervisor/java-cmd) "-cp" classpath
- (str "-Dlogfile.name=" "worker.log")
- "-Dstorm.home="
- (str "-Dworkers.artifacts=" "/tmp/workers-artifacts")
- (str "-Dstorm.id=" mock-storm-id)
- (str "-Dworker.id=" mock-worker-id)
- (str "-Dworker.port=" mock-port)
- "-Dstorm.log.dir=/logs"
- "-Dlog4j.configurationFile=/log4j2/worker.xml"
- "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector"
- "org.apache.storm.LogWriter"]
- [(supervisor/java-cmd) "-server"]
- opts
- topo-opts
- ["-Djava.library.path="
- (str "-Dlogfile.name=" "worker.log")
- "-Dstorm.home="
- "-Dworkers.artifacts=/tmp/workers-artifacts"
- "-Dstorm.conf.file="
- "-Dstorm.options="
- (str "-Dstorm.log.dir=" file-path-separator "logs")
- (str "-Djava.io.tmpdir=" file-path-separator "storm-local" file-path-separator
- "workers" file-path-separator mock-worker-id file-path-separator "tmp")
- (str "-Dlogging.sensitivity=" mock-sensitivity)
- (str "-Dlog4j.configurationFile=" file-path-separator "log4j2" file-path-separator "worker.xml")
- "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector"
- (str "-Dstorm.id=" mock-storm-id)
- (str "-Dworker.id=" mock-worker-id)
- (str "-Dworker.port=" mock-port)
- "-cp" classpath
- "org.apache.storm.daemon.worker"
- mock-storm-id
- mock-port
- mock-worker-id]))]
- (testing "testing *.worker.childopts as strings with extra spaces"
- (let [string-opts "-Dfoo=bar -Xmx1024m"
- topo-string-opts "-Dkau=aux -Xmx2048m"
- exp-args (exp-args-fn ["-Dfoo=bar" "-Xmx1024m"]
- ["-Dkau=aux" "-Xmx2048m"]
- mock-cp)
- mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
- WORKER-CHILDOPTS string-opts}}]
- (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
- topo-string-opts}
- read-supervisor-topology mock-storm-topology
- add-to-classpath mock-cp
- supervisor-stormdist-root nil
- launch-process nil
- set-worker-user! nil
- supervisor/jlp nil
- worker-artifacts-root "/tmp/workers-artifacts"
- supervisor/write-log-metadata! nil
- supervisor/create-blobstore-links nil]
- (supervisor/launch-worker mock-supervisor
- mock-storm-id
- mock-port
- mock-worker-id
- mock-mem-onheap)
- (verify-first-call-args-for-indices launch-process
- [0]
- exp-args))))
- (testing "testing *.worker.childopts as list of strings, with spaces in values"
- (let [list-opts '("-Dopt1='this has a space in it'" "-Xmx1024m")
- topo-list-opts '("-Dopt2='val with spaces'" "-Xmx2048m")
- exp-args (exp-args-fn list-opts topo-list-opts mock-cp)
- mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
- WORKER-CHILDOPTS list-opts}}]
- (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
- topo-list-opts}
- read-supervisor-topology mock-storm-topology
- add-to-classpath mock-cp
- supervisor-stormdist-root nil
- launch-process nil
- set-worker-user! nil
- supervisor/jlp nil
- supervisor/write-log-metadata! nil
- supervisor/create-blobstore-links nil
- worker-artifacts-root "/tmp/workers-artifacts"]
- (supervisor/launch-worker mock-supervisor
- mock-storm-id
- mock-port
- mock-worker-id
- mock-mem-onheap)
- (verify-first-call-args-for-indices launch-process
- [0]
- exp-args))))
- (testing "testing topology.classpath is added to classpath"
- (let [topo-cp (str file-path-separator "any" file-path-separator "path")
- exp-args (exp-args-fn [] [] (add-to-classpath mock-cp [topo-cp]))
- mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed}}]
- (stubbing [read-supervisor-storm-conf {TOPOLOGY-CLASSPATH topo-cp}
- supervisor-stormdist-root nil
- read-supervisor-topology mock-storm-topology
- supervisor/jlp nil
- worker-artifacts-root "/tmp/workers-artifacts"
- set-worker-user! nil
- supervisor/write-log-metadata! nil
- launch-process nil
- current-classpath (str file-path-separator "base")
- supervisor/create-blobstore-links nil]
- (supervisor/launch-worker mock-supervisor
- mock-storm-id
- mock-port
- mock-worker-id
- mock-mem-onheap)
- (verify-first-call-args-for-indices launch-process
- [0]
- exp-args))))
- (testing "testing topology.environment is added to environment for worker launch"
- (let [topo-env {"THISVAR" "somevalue" "THATVAR" "someothervalue"}
- full-env (merge topo-env {"LD_LIBRARY_PATH" nil})
- exp-args (exp-args-fn [] [] mock-cp)
- mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed}}]
- (stubbing [read-supervisor-storm-conf {TOPOLOGY-ENVIRONMENT topo-env}
- supervisor-stormdist-root nil
- read-supervisor-topology mock-storm-topology
- supervisor/jlp nil
- worker-artifacts-root "/tmp/workers-artifacts"
- launch-process nil
- set-worker-user! nil
- supervisor/write-log-metadata! nil
- current-classpath (str file-path-separator "base")
- supervisor/create-blobstore-links nil]
- (supervisor/launch-worker mock-supervisor
- mock-storm-id
- mock-port
- mock-worker-id
- mock-mem-onheap)
- (verify-first-call-args-for-indices launch-process
- [2]
- full-env)))))))
-
-(deftest test-worker-launch-command-run-as-user
- (testing "*.worker.childopts configuration"
- (let [mock-port "42"
- mock-storm-id "fake-storm-id"
- mock-worker-id "fake-worker-id"
- mock-mem-onheap 512
- mock-sensitivity "S3"
- mock-cp "mock-classpath'quote-on-purpose"
- builder (TopologyBuilder.)
- \u2028_ (.setSpout builder "wordSpout" (TestWordSpout.) 1)
- \u2028_ (.shuffleGrouping (.setBolt builder "wordCountBolt" (TestWordCounter.) 1) "wordSpout")
- \u2028mock-storm-topology (.createTopology builder)
- attrs (make-array FileAttribute 0)
- storm-local (.getCanonicalPath (.toFile (Files/createTempDirectory "storm-local" attrs)))
- worker-script (str storm-local "/workers/" mock-worker-id "/storm-worker-script.sh")
- exp-launch ["/bin/worker-launcher"
- "me"
- "worker"
- (str storm-local "/workers/" mock-worker-id)
- worker-script]
- exp-script-fn (fn [opts topo-opts]
- (str "#!/bin/bash\n'export' 'LD_LIBRARY_PATH=';\n\nexec 'java'"
- " '-cp' 'mock-classpath'\"'\"'quote-on-purpose'"
- " '-Dlogfile.name=" "worker.log'"
- " '-Dstorm.home='"
- " '-Dworkers.artifacts=" (str storm-local "/workers-artifacts'")
- " '-Dstorm.id=" mock-storm-id "'"
- " '-Dworker.id=" mock-worker-id "'"
- " '-Dworker.port=" mock-port "'"
- " '-Dstorm.log.dir=/logs'"
- " '-Dlog4j.configurationFile=/log4j2/worker.xml'"
- " '-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector'"
- " 'org.apache.storm.LogWriter'"
- " 'java' '-server'"
- " " (shell-cmd opts)
- " " (shell-cmd topo-opts)
- " '-Djava.library.path='"
- " '-Dlogfile.name=" "worker.log'"
- " '-Dstorm.home='"
- " '-Dworkers.artifacts=" (str storm-local "/workers-artifacts'")
- " '-Dstorm.conf.file='"
- " '-Dstorm.options='"
- " '-Dstorm.log.dir=/logs'"
- " '-Djava.io.tmpdir=" (str storm-local "/workers/" mock-worker-id "/tmp'")
- " '-Dlogging.sensitivity=" mock-sensitivity "'"
- " '-Dlog4j.configurationFile=/log4j2/worker.xml'"
- " '-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector'"
- " '-Dstorm.id=" mock-storm-id "'"
- " '-Dworker.id=" mock-worker-id "'"
- " '-Dworker.port=" mock-port "'"
- " '-cp' 'mock-classpath'\"'\"'quote-on-purpose'"
- " 'org.apache.storm.daemon.worker'"
- " '" mock-storm-id "'"
- " '" mock-port "'"
- " '" mock-worker-id "';"))]
- (try
- (testing "testing *.worker.childopts as strings with extra spaces"
- (let [string-opts "-Dfoo=bar -Xmx1024m"
- topo-string-opts "-Dkau=aux -Xmx2048m"
- exp-script (exp-script-fn ["-Dfoo=bar" "-Xmx1024m"]
- ["-Dkau=aux" "-Xmx2048m"])
- _ (.mkdirs (io/file storm-local "workers" mock-worker-id))
- mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
- STORM-LOCAL-DIR storm-local
- STORM-WORKERS-ARTIFACTS-DIR (str storm-local "/workers-artifacts")
- SUPERVISOR-RUN-WORKER-AS-USER true
- WORKER-CHILDOPTS string-opts}}]
- (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
- topo-string-opts
- TOPOLOGY-SUBMITTER-USER "me"}
- read-supervisor-topology mock-storm-topology
- add-to-classpath mock-cp
- supervisor-stormdist-root nil
- launch-process nil
- set-worker-user! nil
- supervisor/java-cmd "java"
- supervisor/jlp nil
- supervisor/write-log-metadata! nil]
- (supervisor/launch-worker mock-supervisor
- mock-storm-id
- mock-port
- mock-worker-id
- mock-mem-onheap)
- (verify-first-call-args-for-indices launch-process
- [0]
- exp-launch))
- (is (= (slurp worker-script) exp-script))))
- (finally (rmr storm-local)))
- (.mkdirs (io/file storm-local "workers" mock-worker-id))
- (try
- (testing "testing *.worker.childopts as list of strings, with spaces in values"
- (let [list-opts '("-Dopt1='this has a space in it'" "-Xmx1024m")
- topo-list-opts '("-Dopt2='val with spaces'" "-Xmx2048m")
- exp-script (exp-script-fn list-opts topo-list-opts)
- mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
- STORM-LOCAL-DIR storm-local
- STORM-WORKERS-ARTIFACTS-DIR (str storm-local "/workers-artifacts")
- SUPERVISOR-RUN-WORKER-AS-USER true
- WORKER-CHILDOPTS list-opts}}]
- (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
- topo-list-opts
- TOPOLOGY-SUBMITTER-USER "me"}
- read-supervisor-topology mock-storm-topology
- add-to-classpath mock-cp
- supervisor-stormdist-root nil
- launch-process nil
- set-worker-user! nil
- supervisor/java-cmd "java"
- supervisor/jlp nil
- supervisor/write-log-metadata! nil]
- (supervisor/launch-worker mock-supervisor
- mock-storm-id
- mock-port
- mock-worker-id
- mock-mem-onheap)
- (verify-first-call-args-for-indices launch-process
- [0]
- exp-launch))
- (is (= (slurp worker-script) exp-script))))
- (finally (rmr storm-local))))))
-
-(deftest test-workers-go-bananas
- ;; test that multiple workers are started for a port, and test that
- ;; supervisor shuts down propertly (doesn't shutdown the most
- ;; recently launched one, checks heartbeats correctly, etc.)
- )
-
-(deftest downloads-code
- )
-
-(deftest test-stateless
- )
-
-(deftest cleans-up-on-unassign
- ;; TODO just do reassign, and check that cleans up worker states after killing but doesn't get rid of downloaded code
- )
-
-(deftest test-supervisor-data-acls
- (testing "supervisor-data uses correct ACLs"
- (let [scheme "digest"
- digest "storm:thisisapoorpassword"
- auth-conf {STORM-ZOOKEEPER-AUTH-SCHEME scheme
- STORM-ZOOKEEPER-AUTH-PAYLOAD digest}
- expected-acls supervisor/SUPERVISOR-ZK-ACLS
- fake-isupervisor (reify ISupervisor
- (getSupervisorId [this] nil)
- (getAssignmentId [this] nil))]
- (stubbing [uptime-computer nil
- cluster/mk-storm-cluster-state nil
- supervisor-state nil
- local-hostname nil
- mk-timer nil
- supervisor-local-dir nil]
- (supervisor/supervisor-data auth-conf nil fake-isupervisor)
- (verify-call-times-for cluster/mk-storm-cluster-state 1)
- (verify-first-call-args-for-indices cluster/mk-storm-cluster-state [2]
- expected-acls)))))
-
-(deftest test-write-log-metadata
- (testing "supervisor writes correct data to logs metadata file"
- (let [exp-owner "alice"
- exp-worker-id "42"
- exp-storm-id "0123456789"
- exp-port 4242
- exp-logs-users ["bob" "charlie" "daryl"]
- exp-logs-groups ["read-only-group" "special-group"]
- storm-conf {TOPOLOGY-SUBMITTER-USER "alice"
- TOPOLOGY-USERS ["charlie" "bob"]
- TOPOLOGY-GROUPS ["special-group"]
- LOGS-GROUPS ["read-only-group"]
- LOGS-USERS ["daryl"]}
- exp-data {TOPOLOGY-SUBMITTER-USER exp-owner
- "worker-id" exp-worker-id
- LOGS-USERS exp-logs-users
- LOGS-GROUPS exp-logs-groups}
- conf {}]
- (mocking [supervisor/write-log-metadata-to-yaml-file!]
- (supervisor/write-log-metadata! storm-conf exp-owner exp-worker-id
- exp-storm-id exp-port conf)
- (verify-called-once-with-args supervisor/write-log-metadata-to-yaml-file!
- exp-storm-id exp-port exp-data conf)))))
-
-(deftest test-worker-launcher-requires-user
- (testing "worker-launcher throws on blank user"
- (mocking [launch-process]
- (is (thrown-cause-with-msg? java.lang.IllegalArgumentException
- #"(?i).*user cannot be blank.*"
- (supervisor/worker-launcher {} nil ""))))))
-
-(defn found? [sub-str input-str]
- (if (string? input-str)
- (contrib-str/substring? sub-str (str input-str))
- (boolean (some #(contrib-str/substring? sub-str %) input-str))))
-
-(defn not-found? [sub-str input-str]
- (complement (found? sub-str input-str)))
-
-(deftest test-substitute-childopts-happy-path-string
- (testing "worker-launcher replaces ids in childopts"
- (let [worker-id "w-01"
- topology-id "s-01"
- port 9999
- mem-onheap 512
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m -Xmx%HEAP-MEM%m"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m" "-Xmx512m")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
- (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-happy-path-list
- (testing "worker-launcher replaces ids in childopts"
- (let [worker-id "w-01"
- topology-id "s-01"
- port 9999
- mem-onheap 512
- childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m" "-Xmx%HEAP-MEM%m")
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m" "-Xmx512m")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
- (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-happy-path-list-arraylist
- (testing "worker-launcher replaces ids in childopts"
- (let [worker-id "w-01"
- topology-id "s-01"
- port 9999
- mem-onheap 512
- childopts '["-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m" "-Xmx%HEAP-MEM%m"]
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m" "-Xmx512m")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
- (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-topology-id-alone
- (testing "worker-launcher replaces ids in childopts"
- (let [worker-id "w-01"
- topology-id "s-01"
- port 9999
- mem-onheap 512
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%TOPOLOGY-ID%.log"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
- (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-no-keys
- (testing "worker-launcher has no ids to replace in childopts"
- (let [worker-id "w-01"
- topology-id "s-01"
- port 9999
- mem-onheap 512
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
- (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-nil-childopts
- (testing "worker-launcher has nil childopts"
- (let [worker-id "w-01"
- topology-id "s-01"
- port 9999
- mem-onheap 512
- childopts nil
- expected-childopts nil
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
- (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-nil-ids
- (testing "worker-launcher has nil ids"
- (let [worker-id nil
- topology-id "s-01"
- port 9999
- mem-onheap 512
- childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log"
- expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log")
- childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
- (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-retry-read-assignments
- (with-simulated-time-local-cluster [cluster
- :supervisors 0
- :ports-per-supervisor 2
- :daemon-conf {NIMBUS-DO-NOT-REASSIGN true
- NIMBUS-MONITOR-FREQ-SECS 10
- TOPOLOGY-MESSAGE-TIMEOUT-SECS 30
- TOPOLOGY-ACKER-EXECUTORS 0}]
- (letlocals
- (bind sup1 (add-supervisor cluster :id "sup1" :ports [1 2 3 4]))
- (bind topology1 (thrift/mk-topology
- {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 2)}
- {}))
- (bind topology2 (thrift/mk-topology
- {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 2)}
- {}))
- (bind state (:storm-cluster-state cluster))
- (bind changed (capture-changed-workers
- (submit-mocked-assignment
- (:nimbus cluster)
- (:storm-cluster-state cluster)
- "topology1"
- {TOPOLOGY-WORKERS 2}
- topology1
- {1 "1"
- 2 "1"}
- {[1 1] ["sup1" 1]
- [2 2] ["sup1" 2]}
- {["sup1" 1] [0.0 0.0 0.0]
- ["sup1" 2] [0.0 0.0 0.0]
- })
- (submit-mocked-assignment
- (:nimbus cluster)
- (:storm-cluster-state cluster)
- "topology2"
- {TOPOLOGY-WORKERS 2}
- topology2
- {1 "1"
- 2 "1"}
- {[1 1] ["sup1" 1]
- [2 2] ["sup1" 2]}
- {["sup1" 1] [0.0 0.0 0.0]
- ["sup1" 2] [0.0 0.0 0.0]
- })
- ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
- (.rebalance (:nimbus cluster) "topology1" (doto (RebalanceOptions.) (.set_wait_secs 0)))
- ))
- (is (empty? (:launched changed)))
- (bind options (RebalanceOptions.))
- (.set_wait_secs options 0)
- (bind changed (capture-changed-workers
- (.rebalance (:nimbus cluster) "topology2" options)
- (advance-cluster-time cluster 10)
- (heartbeat-workers cluster "sup1" [1 2 3 4])
- (advance-cluster-time cluster 10)
- ))
- (validate-launched-once (:launched changed)
- {"sup1" [1 2]}
- (get-storm-id (:storm-cluster-state cluster) "topology1"))
- (validate-launched-once (:launched changed)
- {"sup1" [3 4]}
- (get-storm-id (:storm-cluster-state cluster) "topology2"))
- )))
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
new file mode 100644
index 0000000..edf534a
--- /dev/null
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
@@ -0,0 +1,483 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.storm.Config;
+import org.apache.storm.daemon.supervisor.Container.ContainerType;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.junit.Test;
+
+public class BasicContainerTest {
+ public static class CommandRun {
+ final List<String> cmd;
+ final Map<String, String> env;
+ final File pwd;
+
+ public CommandRun(List<String> cmd, Map<String, String> env, File pwd) {
+ this.cmd = cmd;
+ this.env = env;
+ this.pwd = pwd;
+ }
+ }
+
+ public static class MockBasicContainer extends BasicContainer {
+ public MockBasicContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+ LocalAssignment assignment, LocalState localState,
+ String workerId, Map<String, Object> topoConf, AdvancedFSOps ops, String profileCmd)
+ throws IOException {
+ super(type, conf, supervisorId, port, assignment, localState, workerId, topoConf, ops,
+ profileCmd);
+ }
+
+ public final List<CommandRun> profileCmds = new ArrayList<>();
+ public final List<CommandRun> workerCmds = new ArrayList<>();
+
+
+ @Override
+ protected Map<String, Object> readTopoConf() throws IOException {
+ return new HashMap<>();
+ }
+
+ @Override
+ public void createNewWorkerId() {
+ super.createNewWorkerId();
+ }
+
+ @Override
+ public List<String> substituteChildopts(Object value, int memOnheap) {
+ return super.substituteChildopts(value, memOnheap);
+ }
+
+ @Override
+ protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix,
+ File targetDir) throws IOException, InterruptedException {
+ profileCmds.add(new CommandRun(command, env, targetDir));
+ return true;
+ }
+
+ @Override
+ protected void launchWorkerProcess(List<String> command, Map<String, String> env, String logPrefix,
+ ExitCodeCallback processExitCallback, File targetDir) throws IOException {
+ workerCmds.add(new CommandRun(command, env, targetDir));
+ }
+
+ @Override
+ protected String javaCmd(String cmd) {
+ //avoid system dependent things
+ return cmd;
+ }
+
+ @Override
+ protected List<String> frameworkClasspath() {
+ //We are not really running anything so make this
+ // simple to check for
+ return Arrays.asList("FRAMEWORK_CP");
+ }
+
+ @Override
+ protected String javaLibraryPath(String stormRoot, Map<String, Object> conf) {
+ return "JLP";
+ }
+ }
+
+ @Test
+ public void testCreateNewWorkerId() throws Exception {
+ final String topoId = "test_topology";
+ final int port = 8080;
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+
+ Map<String, Object> superConf = new HashMap<>();
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+
+ LocalState ls = mock(LocalState.class);
+
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", port, la, ls, null, new HashMap<>(), ops, "profile");
+ //null worker id means generate one...
+
+ assertNotNull(mc._workerId);
+ verify(ls).getApprovedWorkers();
+ Map<String, Integer> expectedNewState = new HashMap<String, Integer>();
+ expectedNewState.put(mc._workerId, port);
+ verify(ls).setApprovedWorkers(expectedNewState);
+ }
+
+ @Test
+ public void testRecovery() throws Exception {
+ final String topoId = "test_topology";
+ final String workerId = "myWorker";
+ final int port = 8080;
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+
+ Map<String, Integer> workerState = new HashMap<String, Integer>();
+ workerState.put(workerId, port);
+
+ LocalState ls = mock(LocalState.class);
+ when(ls.getApprovedWorkers()).thenReturn(workerState);
+
+ Map<String, Object> superConf = new HashMap<>();
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.RECOVER_FULL, superConf,
+ "SUPERVISOR", port, la, ls, null, new HashMap<>(), ops, "profile");
+
+ assertEquals(workerId, mc._workerId);
+ }
+
+ @Test
+ public void testRecoveryMiss() throws Exception {
+ final String topoId = "test_topology";
+ final int port = 8080;
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+
+ Map<String, Integer> workerState = new HashMap<String, Integer>();
+ workerState.put("somethingelse", port+1);
+
+ LocalState ls = mock(LocalState.class);
+ when(ls.getApprovedWorkers()).thenReturn(workerState);
+
+ try {
+ new MockBasicContainer(ContainerType.RECOVER_FULL, new HashMap<String, Object>(),
+ "SUPERVISOR", port, la, ls, null, new HashMap<>(), null, "profile");
+ fail("Container recovered worker incorrectly");
+ } catch (ContainerRecoveryException e) {
+ //Expected
+ }
+ }
+
+ @Test
+ public void testCleanUp() throws Exception {
+ final String topoId = "test_topology";
+ final int port = 8080;
+ final String workerId = "worker-id";
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+
+ Map<String, Object> superConf = new HashMap<>();
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+
+ Map<String, Integer> workerState = new HashMap<String, Integer>();
+ workerState.put(workerId, port);
+
+ LocalState ls = mock(LocalState.class);
+ when(ls.getApprovedWorkers()).thenReturn(new HashMap<>(workerState));
+
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+
+ mc.cleanUp();
+
+ assertNull(mc._workerId);
+ verify(ls).getApprovedWorkers();
+ Map<String, Integer> expectedNewState = new HashMap<String, Integer>();
+ verify(ls).setApprovedWorkers(expectedNewState);
+ }
+
+ @Test
+ public void testRunProfiling() throws Exception {
+ final long pid = 100;
+ final String topoId = "test_topology";
+ final int port = 8080;
+ final String workerId = "worker-id";
+ final String stormLocal = ContainerTest.asAbsPath("tmp", "testing");
+ final String topoRoot = ContainerTest.asAbsPath(stormLocal, topoId, String.valueOf(port));
+ final File workerArtifactsPid = ContainerTest.asAbsFile(topoRoot, "worker.pid");
+
+ final Map<String, Object> superConf = new HashMap<>();
+ superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
+ superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
+
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+ when(ops.slurpString(workerArtifactsPid)).thenReturn(String.valueOf(pid));
+
+ LocalState ls = mock(LocalState.class);
+
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+
+ //HEAP DUMP
+ ProfileRequest req = new ProfileRequest();
+ req.set_action(ProfileAction.JMAP_DUMP);
+
+ mc.runProfiling(req, false);
+
+ assertEquals(1, mc.profileCmds.size());
+ CommandRun cmd = mc.profileCmds.get(0);
+ mc.profileCmds.clear();
+ assertEquals(Arrays.asList("profile", String.valueOf(pid), "jmap", topoRoot), cmd.cmd);
+ assertEquals(new File(topoRoot), cmd.pwd);
+
+ //JSTACK DUMP
+ req.set_action(ProfileAction.JSTACK_DUMP);
+
+ mc.runProfiling(req, false);
+
+ assertEquals(1, mc.profileCmds.size());
+ cmd = mc.profileCmds.get(0);
+ mc.profileCmds.clear();
+ assertEquals(Arrays.asList("profile", String.valueOf(pid), "jstack", topoRoot), cmd.cmd);
+ assertEquals(new File(topoRoot), cmd.pwd);
+
+ //RESTART
+ req.set_action(ProfileAction.JVM_RESTART);
+
+ mc.runProfiling(req, false);
+
+ assertEquals(1, mc.profileCmds.size());
+ cmd = mc.profileCmds.get(0);
+ mc.profileCmds.clear();
+ assertEquals(Arrays.asList("profile", String.valueOf(pid), "kill"), cmd.cmd);
+ assertEquals(new File(topoRoot), cmd.pwd);
+
+ //JPROFILE DUMP
+ req.set_action(ProfileAction.JPROFILE_DUMP);
+
+ mc.runProfiling(req, false);
+
+ assertEquals(1, mc.profileCmds.size());
+ cmd = mc.profileCmds.get(0);
+ mc.profileCmds.clear();
+ assertEquals(Arrays.asList("profile", String.valueOf(pid), "dump", topoRoot), cmd.cmd);
+ assertEquals(new File(topoRoot), cmd.pwd);
+
+ //JPROFILE START
+ req.set_action(ProfileAction.JPROFILE_STOP);
+
+ mc.runProfiling(req, false);
+
+ assertEquals(1, mc.profileCmds.size());
+ cmd = mc.profileCmds.get(0);
+ mc.profileCmds.clear();
+ assertEquals(Arrays.asList("profile", String.valueOf(pid), "start"), cmd.cmd);
+ assertEquals(new File(topoRoot), cmd.pwd);
+
+ //JPROFILE STOP
+ req.set_action(ProfileAction.JPROFILE_STOP);
+
+ mc.runProfiling(req, true);
+
+ assertEquals(1, mc.profileCmds.size());
+ cmd = mc.profileCmds.get(0);
+ mc.profileCmds.clear();
+ assertEquals(Arrays.asList("profile", String.valueOf(pid), "stop", topoRoot), cmd.cmd);
+ assertEquals(new File(topoRoot), cmd.pwd);
+ }
+
+ private static void setSystemProp(String key, String value) {
+ if (value == null) {
+ System.clearProperty(key);
+ } else {
+ System.setProperty(key, value);
+ }
+ }
+
+ private static interface Run {
+ public void run() throws Exception;
+ }
+
+ private static void checkpoint(Run r, String ... newValues) throws Exception {
+ if (newValues.length % 2 != 0) {
+ throw new IllegalArgumentException("Parameters are of the form system property name, new value");
+ }
+ Map<String, String> orig = new HashMap<>();
+ try {
+ for (int index = 0; index < newValues.length; index += 2) {
+ String key = newValues[index];
+ String value = newValues[index + 1];
+ orig.put(key, System.getProperty(key));
+ setSystemProp(key, value);
+ }
+ r.run();
+ } finally {
+ for (Map.Entry<String, String> entry: orig.entrySet()) {
+ setSystemProp(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ private static <T> void assertListEquals(List<T> a, List<T> b) {
+ if (a == null) {
+ assertNull(b);
+ }
+ if (b == null) {
+ assertNull(a);
+ }
+ int commonLen = Math.min(a.size(), b.size());
+ for (int i = 0; i < commonLen; i++) {
+ assertEquals("at index "+i+"\n"+a+" !=\n"+b+"\n", a.get(i), b.get(i));
+ }
+
+ assertEquals("size of lists don't match \n"+a+" !=\n"+b, a.size(), b.size());
+ }
+
+ @Test
+ public void testLaunch() throws Exception {
+ final String topoId = "test_topology";
+ final int port = 8080;
+ final String stormHome = ContainerTest.asAbsPath("tmp", "storm-home");
+ final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath();
+ final String workerId = "worker-id";
+ final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local");
+ final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", topoId);
+ final File stormcode = new File(distRoot, "stormcode.ser");
+ final File stormjar = new File(distRoot, "stormjar.jar");
+ final String log4jdir = ContainerTest.asAbsPath(stormHome, "conf");
+ final String workerConf = ContainerTest.asAbsPath(log4jdir, "worker.xml");
+ final String workerRoot = ContainerTest.asAbsPath(stormLocal, "workers", workerId);
+ final String workerTmpDir = ContainerTest.asAbsPath(workerRoot, "tmp");
+
+ final StormTopology st = new StormTopology();
+ st.set_spouts(new HashMap<>());
+ st.set_bolts(new HashMap<>());
+ st.set_state_spouts(new HashMap<>());
+ byte [] serializedState = Utils.gzip(Utils.thriftSerialize(st));
+
+ final Map<String, Object> superConf = new HashMap<>();
+ superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
+ superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
+ superConf.put(Config.STORM_LOG4J2_CONF_DIR, log4jdir);
+
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+ when(ops.slurp(stormcode)).thenReturn(serializedState);
+
+ LocalState ls = mock(LocalState.class);
+
+
+ checkpoint(() -> {
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+
+ mc.launch();
+
+ assertEquals(1, mc.workerCmds.size());
+ CommandRun cmd = mc.workerCmds.get(0);
+ mc.workerCmds.clear();
+ assertListEquals(Arrays.asList(
+ "java",
+ "-cp",
+ "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
+ "-Dlogging.sensitivity=S3",
+ "-Dlogfile.name=worker.log",
+ "-Dstorm.home=" + stormHome,
+ "-Dworkers.artifacts=" + stormLocal,
+ "-Dstorm.id=" + topoId,
+ "-Dworker.id=" + workerId,
+ "-Dworker.port=" + port,
+ "-Dstorm.log.dir=" + stormLogDir,
+ "-Dlog4j.configurationFile=" + workerConf,
+ "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
+ "-Dstorm.local.dir=" + stormLocal,
+ "org.apache.storm.LogWriter",
+ "java",
+ "-server",
+ "-Dlogging.sensitivity=S3",
+ "-Dlogfile.name=worker.log",
+ "-Dstorm.home=" + stormHome,
+ "-Dworkers.artifacts=" + stormLocal,
+ "-Dstorm.id=" + topoId,
+ "-Dworker.id=" + workerId,
+ "-Dworker.port=" + port,
+ "-Dstorm.log.dir=" + stormLogDir,
+ "-Dlog4j.configurationFile=" + workerConf,
+ "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
+ "-Dstorm.local.dir=" + stormLocal,
+ "-Djava.library.path=JLP",
+ "-Dstorm.conf.file=",
+ "-Dstorm.options=",
+ "-Djava.io.tmpdir="+workerTmpDir,
+ "-cp",
+ "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
+ "org.apache.storm.daemon.worker",
+ topoId,
+ "SUPERVISOR",
+ String.valueOf(port),
+ workerId
+ ), cmd.cmd);
+ assertEquals(new File(workerRoot), cmd.pwd);
+ },
+ "storm.home", stormHome,
+ "storm.log.dir", stormLogDir);
+ }
+
+ @Test
+ public void testSubstChildOpts() throws Exception {
+ String workerId = "w-01";
+ String topoId = "s-01";
+ int port = 9999;
+ int memOnheap = 512;
+
+ LocalAssignment la = new LocalAssignment();
+ la.set_topology_id(topoId);
+
+ Map<String, Object> superConf = new HashMap<>();
+
+ AdvancedFSOps ops = mock(AdvancedFSOps.class);
+ when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+
+ LocalState ls = mock(LocalState.class);
+
+ MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf,
+ "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+
+ assertListEquals(Arrays.asList(
+ "-Xloggc:/tmp/storm/logs/gc.worker-9999-s-01-w-01-9999.log",
+ "-Xms256m",
+ "-Xmx512m"),
+ mc.substituteChildopts("-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m -Xmx%HEAP-MEM%m", memOnheap));
+
+ assertListEquals(Arrays.asList(
+ "-Xloggc:/tmp/storm/logs/gc.worker-9999-s-01-w-01-9999.log",
+ "-Xms256m",
+ "-Xmx512m"),
+ mc.substituteChildopts(Arrays.asList("-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log","-Xms256m","-Xmx%HEAP-MEM%m"), memOnheap));
+
+ assertListEquals(Collections.emptyList(),
+ mc.substituteChildopts(null));
+ }
+}
[11/21] storm git commit: Merge branch 'STORM-2117' of
https://github.com/HeartSaVioR/storm into STORM-2117
Posted by ka...@apache.org.
Merge branch 'STORM-2117' of https://github.com/HeartSaVioR/storm into STORM-2117
STROM-2117: Supervisor V2 with local mode extracts resources directory to the wrong directory
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fb42b960
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fb42b960
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fb42b960
Branch: refs/heads/1.x-branch
Commit: fb42b960fae0a06f13f2ee83ddf2344521c1c8e8
Parents: 24ff529
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Sep 23 12:28:03 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Sep 23 12:58:06 2016 -0500
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/localizer/AsyncLocalizer.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/fb42b960/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java b/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
index 7887281..d3e3925 100644
--- a/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
+++ b/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
@@ -192,12 +192,12 @@ public class AsyncLocalizer implements ILocalizer, Shutdownable {
if (resourcesJar != null) {
LOG.info("Extracting resources from jar at {} to {}", resourcesJar, targetDir);
- Utils.extractDirFromJar(resourcesJar, ConfigUtils.RESOURCES_SUBDIR, _stormRoot);
+ Utils.extractDirFromJar(resourcesJar, ConfigUtils.RESOURCES_SUBDIR, new File(targetDir));
} else if (url != null) {
LOG.info("Copying resources at {} to {} ", url.toString(), targetDir);
if ("jar".equals(url.getProtocol())) {
JarURLConnection urlConnection = (JarURLConnection) url.openConnection();
- Utils.extractDirFromJar(urlConnection.getJarFileURL().getFile(), ConfigUtils.RESOURCES_SUBDIR, _stormRoot);
+ Utils.extractDirFromJar(urlConnection.getJarFileURL().getFile(), ConfigUtils.RESOURCES_SUBDIR, new File(targetDir));
} else {
_fsOps.copyDirectory(new File(url.getFile()), new File(targetDir));
}
[08/21] storm git commit: STORM-2018: Supervisor V2
Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/testing.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/testing.clj b/storm-core/src/clj/org/apache/storm/testing.clj
index 565d1b9..4029be1 100644
--- a/storm-core/src/clj/org/apache/storm/testing.clj
+++ b/storm-core/src/clj/org/apache/storm/testing.clj
@@ -17,12 +17,13 @@
(ns org.apache.storm.testing
(:require [org.apache.storm.daemon
[nimbus :as nimbus]
- [supervisor :as supervisor]
+ [local-supervisor :as local-supervisor]
[common :as common]
[worker :as worker]
[executor :as executor]])
- (:require [org.apache.storm [process-simulator :as psim]])
- (:import [org.apache.commons.io FileUtils])
+ (:import [org.apache.commons.io FileUtils]
+ [org.apache.storm ProcessSimulator]
+ [org.apache.storm.daemon.supervisor Supervisor StandaloneSupervisor SupervisorUtils])
(:import [java.io File])
(:import [java.util HashMap ArrayList])
(:import [java.util.concurrent.atomic AtomicInteger])
@@ -31,6 +32,7 @@
(:import [org.apache.storm.tuple Fields Tuple TupleImpl])
(:import [org.apache.storm.task TopologyContext])
(:import [org.apache.storm.generated GlobalStreamId Bolt KillOptions])
+ (:import [org.apache.storm.messaging IContext])
(:import [org.apache.storm.testing FeederSpout FixedTupleSpout FixedTuple
TupleCaptureBolt SpoutTracker BoltTracker NonRichBoltTracker
TestWordSpout MemoryTransactionalSpout])
@@ -108,8 +110,10 @@
conf
{STORM-LOCAL-DIR tmp-dir
SUPERVISOR-SLOTS-PORTS port-ids})
- id-fn (if id (fn [] id) supervisor/generate-supervisor-id)
- daemon (with-var-roots [supervisor/generate-supervisor-id id-fn] (supervisor/mk-supervisor supervisor-conf (:shared-context cluster-map) (supervisor/standalone-supervisor)))]
+ id (or id (Utils/uuid))
+ isupervisor (proxy [StandaloneSupervisor] []
+ (generateSupervisorId [] id))
+ daemon (local-supervisor/mk-local-supervisor supervisor-conf (:shared-context cluster-map) isupervisor)]
(swap! (:supervisors cluster-map) conj daemon)
(swap! (:tmp-dirs cluster-map) conj tmp-dir)
daemon))
@@ -135,18 +139,19 @@
(let [zk-tmp (local-temp-path)
[zk-port zk-handle] (if-not (contains? daemon-conf STORM-ZOOKEEPER-SERVERS)
(zk/mk-inprocess-zookeeper zk-tmp))
+ nimbus-tmp (local-temp-path)
daemon-conf (merge (read-storm-config)
{TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true
ZMQ-LINGER-MILLIS 0
TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS false
TOPOLOGY-TRIDENT-BATCH-EMIT-INTERVAL-MILLIS 50
STORM-CLUSTER-MODE "local"
- BLOBSTORE-SUPERUSER (System/getProperty "user.name")}
+ BLOBSTORE-SUPERUSER (System/getProperty "user.name")
+ BLOBSTORE-DIR nimbus-tmp}
(if-not (contains? daemon-conf STORM-ZOOKEEPER-SERVERS)
{STORM-ZOOKEEPER-PORT zk-port
STORM-ZOOKEEPER-SERVERS ["localhost"]})
daemon-conf)
- nimbus-tmp (local-temp-path)
port-counter (mk-counter supervisor-slot-port-min)
nimbus (nimbus/service-handler
(assoc daemon-conf STORM-LOCAL-DIR nimbus-tmp)
@@ -172,17 +177,17 @@
cluster-map))
(defn get-supervisor [cluster-map supervisor-id]
- (let [finder-fn #(= (.get-id %) supervisor-id)]
+ (let [finder-fn #(= (.getId %) supervisor-id)]
(find-first finder-fn @(:supervisors cluster-map))))
(defn kill-supervisor [cluster-map supervisor-id]
- (let [finder-fn #(= (.get-id %) supervisor-id)
+ (let [finder-fn #(= (.getId %) supervisor-id)
supervisors @(:supervisors cluster-map)
sup (find-first finder-fn
supervisors)]
;; tmp-dir will be taken care of by shutdown
(reset! (:supervisors cluster-map) (remove-first finder-fn supervisors))
- (.shutdown sup)))
+ (.close sup)))
(defn kill-local-storm-cluster [cluster-map]
(.shutdown (:nimbus cluster-map))
@@ -196,10 +201,10 @@
(.close (:state cluster-map))
(.disconnect (:storm-cluster-state cluster-map))
(doseq [s @(:supervisors cluster-map)]
- (.shutdown-all-workers s)
+ (.shutdownAllWorkers s)
;; race condition here? will it launch the workers again?
- (supervisor/kill-supervisor s))
- (psim/kill-all-processes)
+ (.close s))
+ (ProcessSimulator/killAllProcesses)
(if (not-nil? (:zookeeper cluster-map))
(do
(log-message "Shutting down in process zookeeper")
@@ -235,19 +240,23 @@
(while-timeout timeout-ms (not (apredicate))
(Time/sleep 100))))
+(defn is-supervisor-waiting [^Supervisor supervisor]
+ (.isWaiting supervisor))
+
+
(defn wait-until-cluster-waiting
"Wait until the cluster is idle. Should be used with time simulation."
([cluster-map] (wait-until-cluster-waiting cluster-map TEST-TIMEOUT-MS))
([cluster-map timeout-ms]
;; wait until all workers, supervisors, and nimbus is waiting
(let [supervisors @(:supervisors cluster-map)
- workers (filter (partial satisfies? common/DaemonCommon) (psim/all-processes))
+ workers (filter (partial satisfies? common/DaemonCommon) (clojurify-structure (ProcessSimulator/getAllProcessHandles)))
daemons (concat
[(:nimbus cluster-map)]
- supervisors
; because a worker may already be dead
workers)]
- (while-timeout timeout-ms (not (every? (memfn waiting?) daemons))
+ (while-timeout timeout-ms (or (not (every? (memfn waiting?) daemons))
+ (not (every? is-supervisor-waiting supervisors)))
(Thread/sleep (rand-int 20))
;; (doseq [d daemons]
;; (if-not ((memfn waiting?) d)
@@ -278,7 +287,8 @@
f# (future (while @keep-waiting?# (simulate-wait ~cluster-sym)))]
(kill-local-storm-cluster ~cluster-sym)
(reset! keep-waiting?# false)
- @f#)))))
+ @f#
+ (.setSubject (ReqContext/context) nil))))))
(defmacro with-simulated-time-local-cluster
[& args]
@@ -360,35 +370,6 @@
worker->port (ls-approved-workers supervisor-state)]
(worker->port worker-id)))
-(defn mk-capture-shutdown-fn
- [capture-atom]
- (let [existing-fn supervisor/shutdown-worker]
- (fn [supervisor worker-id]
- (let [conf (:conf supervisor)
- supervisor-id (:supervisor-id supervisor)
- port (find-worker-port conf worker-id)
- existing (get @capture-atom [supervisor-id port] 0)]
- (swap! capture-atom assoc [supervisor-id port] (inc existing))
- (existing-fn supervisor worker-id)))))
-
-(defmacro capture-changed-workers
- [& body]
- `(let [launch-captured# (atom {})
- shutdown-captured# (atom {})]
- (with-var-roots [supervisor/launch-worker (mk-capture-launch-fn launch-captured#)
- supervisor/shutdown-worker (mk-capture-shutdown-fn shutdown-captured#)]
- ~@body
- {:launched @launch-captured#
- :shutdown @shutdown-captured#})))
-
-(defmacro capture-launched-workers
- [& body]
- `(:launched (capture-changed-workers ~@body)))
-
-(defmacro capture-shutdown-workers
- [& body]
- `(:shutdown (capture-changed-workers ~@body)))
-
(defnk aggregated-stat
[cluster-map storm-name stat-key :component-ids nil]
(let [state (:storm-cluster-state cluster-map)
@@ -660,6 +641,7 @@
;; (println "Spout emitted: " (global-amt track-id "spout-emitted"))
;; (println "Processed: " (global-amt track-id "processed"))
;; (println "Transferred: " (global-amt track-id "transferred"))
+ (advance-time-secs! 1)
(Thread/sleep (rand-int 200)))
(reset! (:last-spout-emit tracked-topology) target))))
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/ProcessSimulator.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/ProcessSimulator.java b/storm-core/src/jvm/org/apache/storm/ProcessSimulator.java
new file mode 100644
index 0000000..202df12
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/ProcessSimulator.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm;
+import org.apache.storm.daemon.Shutdownable;
+
+import java.util.Collection;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * In local mode, {@code ProcessSimulator} keeps track of Shutdownable objects
+ * in place of actual processes (in cluster mode).
+ */
+public class ProcessSimulator {
+ private static Logger LOG = LoggerFactory.getLogger(ProcessSimulator.class);
+ private static Object lock = new Object();
+ protected static ConcurrentHashMap<String, Shutdownable> processMap = new ConcurrentHashMap<String, Shutdownable>();
+
+ /**
+ * Register a process' handle
+ *
+ * @param pid
+ * @param shutdownable
+ */
+ public static void registerProcess(String pid, Shutdownable shutdownable) {
+ processMap.put(pid, shutdownable);
+ }
+
+ /**
+ * Get all process handles
+ *
+ * @return
+ */
+ public static Collection<Shutdownable> getAllProcessHandles() {
+ return processMap.values();
+ }
+
+ /**
+ * Kill a process
+ *
+ * @param pid
+ */
+ public static void killProcess(String pid) {
+ synchronized (lock) {
+ LOG.info("Begin killing process " + pid);
+ Shutdownable shutdownHandle = processMap.get(pid);
+ if (shutdownHandle != null) {
+ shutdownHandle.shutdown();
+ }
+ processMap.remove(pid);
+ LOG.info("Successfully killed process " + pid);
+ }
+ }
+
+ /**
+ * Kill all processes
+ */
+ public static void killAllProcesses() {
+ Set<String> pids = processMap.keySet();
+ for (String pid : pids) {
+ killProcess(pid);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/StormTimer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/StormTimer.java b/storm-core/src/jvm/org/apache/storm/StormTimer.java
new file mode 100644
index 0000000..4be62b4
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/StormTimer.java
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm;
+
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Comparator;
+import java.util.Random;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * The timer defined in this file is very similar to java.util.Timer, except
+ * it integrates with Storm's time simulation capabilities. This lets us test
+ * code that does asynchronous work on the timer thread
+ */
+
+public class StormTimer implements AutoCloseable {
+ private static final Logger LOG = LoggerFactory.getLogger(StormTimer.class);
+
+ public static class QueueEntry {
+ public final Long endTimeMs;
+ public final Runnable func;
+ public final String id;
+
+ public QueueEntry(Long endTimeMs, Runnable func, String id) {
+ this.endTimeMs = endTimeMs;
+ this.func = func;
+ this.id = id;
+ }
+ }
+
+ public static class StormTimerTask extends Thread {
+
+ //initialCapacity set to 11 since its the default inital capacity of PriorityBlockingQueue
+ private PriorityBlockingQueue<QueueEntry> queue = new PriorityBlockingQueue<QueueEntry>(11, new Comparator<QueueEntry>() {
+ @Override
+ public int compare(QueueEntry o1, QueueEntry o2) {
+ return o1.endTimeMs.intValue() - o2.endTimeMs.intValue();
+ }
+ });
+
+ // boolean to indicate whether timer is active
+ private AtomicBoolean active = new AtomicBoolean(false);
+
+ // function to call when timer is killed
+ private Thread.UncaughtExceptionHandler onKill;
+
+ //random number generator
+ private Random random = new Random();
+
+ @Override
+ public void run() {
+ while (this.active.get()) {
+ QueueEntry queueEntry = null;
+ try {
+ queueEntry = this.queue.peek();
+ if ((queueEntry != null) && (Time.currentTimeMillis() >= queueEntry.endTimeMs)) {
+ // It is imperative to not run the function
+ // inside the timer lock. Otherwise, it is
+ // possible to deadlock if the fn deals with
+ // other locks, like the submit lock.
+ this.queue.remove(queueEntry);
+ queueEntry.func.run();
+ } else if (queueEntry != null) {
+ // If any events are scheduled, sleep until
+ // event generation. If any recurring events
+ // are scheduled then we will always go
+ // through this branch, sleeping only the
+ // exact necessary amount of time. We give
+ // an upper bound, e.g. 1000 millis, to the
+ // sleeping time, to limit the response time
+ // for detecting any new event within 1 secs.
+ Time.sleep(Math.min(1000, (queueEntry.endTimeMs - Time.currentTimeMillis())));
+ } else {
+ // Otherwise poll to see if any new event
+ // was scheduled. This is, in essence, the
+ // response time for detecting any new event
+ // schedulings when there are no scheduled
+ // events.
+ Time.sleep(1000);
+ }
+ } catch (Throwable e) {
+ if (!(Utils.exceptionCauseIsInstanceOf(InterruptedException.class, e))) {
+ this.onKill.uncaughtException(this, e);
+ this.setActive(false);
+ }
+ }
+ }
+ }
+
+ public void setOnKillFunc(Thread.UncaughtExceptionHandler onKill) {
+ this.onKill = onKill;
+ }
+
+ public void setActive(boolean flag) {
+ this.active.set(flag);
+ }
+
+ public boolean isActive() {
+ return this.active.get();
+ }
+
+ public void add(QueueEntry queueEntry) {
+ this.queue.add(queueEntry);
+ }
+ }
+
+ //task to run
+ private StormTimerTask task = new StormTimerTask();
+
+ /**
+ * Makes a Timer in the form of a StormTimerTask Object
+ * @param name name of the timer
+ * @param onKill function to call when timer is killed unexpectedly
+ * @return StormTimerTask object that was initialized
+ */
+ public StormTimer (String name, Thread.UncaughtExceptionHandler onKill) {
+ if (onKill == null) {
+ throw new RuntimeException("onKill func is null!");
+ }
+ if (name == null) {
+ this.task.setName("timer");
+ } else {
+ this.task.setName(name);
+ }
+ this.task.setOnKillFunc(onKill);
+ this.task.setActive(true);
+
+ this.task.setDaemon(true);
+ this.task.setPriority(Thread.MAX_PRIORITY);
+ this.task.start();
+ }
+
+ /**
+ * Schedule a function to be executed in the timer
+ * @param delaySecs the number of seconds to delay before running the function
+ * @param func the function to run
+ * @param checkActive whether to check is the timer is active
+ * @param jitterMs add jitter to the run
+ */
+ public void schedule(int delaySecs, Runnable func, boolean checkActive, int jitterMs) {
+ if (func == null) {
+ throw new RuntimeException("function to schedule is null!");
+ }
+ if (checkActive) {
+ checkActive();
+ }
+ String id = Utils.uuid();
+ long endTimeMs = Time.currentTimeMillis() + Time.secsToMillisLong(delaySecs);
+ if (jitterMs > 0) {
+ endTimeMs = this.task.random.nextInt(jitterMs) + endTimeMs;
+ }
+ task.add(new QueueEntry(endTimeMs, func, id));
+ }
+
+ public void schedule(int delaySecs, Runnable func) {
+ schedule(delaySecs, func, true, 0);
+ }
+
+ /**
+ * Schedule a function to run recurrently
+ * @param delaySecs the number of seconds to delay before running the function
+ * @param recurSecs the time between each invocation
+ * @param func the function to run
+ */
+ public void scheduleRecurring(int delaySecs, final int recurSecs, final Runnable func) {
+ schedule(delaySecs, new Runnable() {
+ @Override
+ public void run() {
+ func.run();
+ // This avoids a race condition with cancel-timer.
+ schedule(recurSecs, this, false, 0);
+ }
+ });
+ }
+
+ /**
+ * schedule a function to run recurrently with jitter
+ * @param delaySecs the number of seconds to delay before running the function
+ * @param recurSecs the time between each invocation
+ * @param jitterMs jitter added to the run
+ * @param func the function to run
+ */
+ public void scheduleRecurringWithJitter(int delaySecs, final int recurSecs, final int jitterMs, final Runnable func) {
+ schedule(delaySecs, new Runnable() {
+ @Override
+ public void run() {
+ func.run();
+ // This avoids a race condition with cancel-timer.
+ schedule(recurSecs, this, false, jitterMs);
+ }
+ });
+ }
+
+ /**
+ * check if timer is active
+ */
+ private void checkActive() {
+ if (!this.task.isActive()) {
+ throw new IllegalStateException("Timer is not active");
+ }
+ }
+
+ /**
+ * cancel timer
+ */
+
+ @Override
+ public void close() throws Exception {
+ checkActive();
+ this.task.setActive(false);
+ this.task.interrupt();
+ this.task.join();
+ }
+
+ /**
+ * is timer waiting. Used in timer simulation
+ */
+ public boolean isTimerWaiting() {
+ return Time.isThreadWaiting(task);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java b/storm-core/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java
new file mode 100644
index 0000000..043dd0c
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.callback;
+
+import org.apache.storm.zookeeper.ZkEventTypes;
+import org.apache.storm.zookeeper.ZkKeeperStates;
+import org.apache.zookeeper.Watcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DefaultWatcherCallBack implements WatcherCallBack {
+
+ private static Logger LOG = LoggerFactory.getLogger(DefaultWatcherCallBack.class);
+
+ @Override
+ public void execute(Watcher.Event.KeeperState state, Watcher.Event.EventType type, String path) {
+ LOG.debug("Zookeeper state update: {}, {}, {}", ZkKeeperStates.getStateName(state), ZkEventTypes.getTypeName(type), path);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/callback/WatcherCallBack.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/callback/WatcherCallBack.java b/storm-core/src/jvm/org/apache/storm/callback/WatcherCallBack.java
new file mode 100644
index 0000000..41a50ec
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/callback/WatcherCallBack.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.callback;
+
+import org.apache.zookeeper.Watcher;
+
+public interface WatcherCallBack {
+ public void execute(Watcher.Event.KeeperState state, Watcher.Event.EventType type, String path);
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java b/storm-core/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java
new file mode 100644
index 0000000..75b0e99
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.callback;
+
+import org.apache.zookeeper.Watcher;
+
+public interface ZKStateChangedCallback {
+ public void changed(Watcher.Event.EventType type, String path);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/ClusterStateFactory.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/ClusterStateFactory.java b/storm-core/src/jvm/org/apache/storm/cluster/ClusterStateFactory.java
index 6474d82..fcf5db0 100644
--- a/storm-core/src/jvm/org/apache/storm/cluster/ClusterStateFactory.java
+++ b/storm-core/src/jvm/org/apache/storm/cluster/ClusterStateFactory.java
@@ -21,7 +21,9 @@ import clojure.lang.APersistentMap;
import java.util.List;
import org.apache.zookeeper.data.ACL;
-public interface ClusterStateFactory {
+import org.apache.storm.cluster.StateStorageFactory;
+
+public interface ClusterStateFactory extends StateStorageFactory {
ClusterState mkState(APersistentMap config, APersistentMap auth_conf, List<ACL> acls, ClusterStateContext context);
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/ClusterUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/ClusterUtils.java b/storm-core/src/jvm/org/apache/storm/cluster/ClusterUtils.java
new file mode 100644
index 0000000..96c177b
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/ClusterUtils.java
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import org.apache.storm.Config;
+import org.apache.storm.generated.ClusterWorkerHeartbeat;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.ExecutorStats;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.utils.Utils;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ClusterUtils {
+
+ public static final String ZK_SEPERATOR = "/";
+
+ public static final String ASSIGNMENTS_ROOT = "assignments";
+ public static final String CODE_ROOT = "code";
+ public static final String STORMS_ROOT = "storms";
+ public static final String SUPERVISORS_ROOT = "supervisors";
+ public static final String WORKERBEATS_ROOT = "workerbeats";
+ public static final String BACKPRESSURE_ROOT = "backpressure";
+ public static final String ERRORS_ROOT = "errors";
+ public static final String BLOBSTORE_ROOT = "blobstore";
+ public static final String BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_ROOT = "blobstoremaxkeysequencenumber";
+ public static final String NIMBUSES_ROOT = "nimbuses";
+ public static final String CREDENTIALS_ROOT = "credentials";
+ public static final String LOGCONFIG_ROOT = "logconfigs";
+ public static final String PROFILERCONFIG_ROOT = "profilerconfigs";
+
+ public static final String ASSIGNMENTS_SUBTREE = ZK_SEPERATOR + ASSIGNMENTS_ROOT;
+ public static final String STORMS_SUBTREE = ZK_SEPERATOR + STORMS_ROOT;
+ public static final String SUPERVISORS_SUBTREE = ZK_SEPERATOR + SUPERVISORS_ROOT;
+ public static final String WORKERBEATS_SUBTREE = ZK_SEPERATOR + WORKERBEATS_ROOT;
+ public static final String BACKPRESSURE_SUBTREE = ZK_SEPERATOR + BACKPRESSURE_ROOT;
+ public static final String ERRORS_SUBTREE = ZK_SEPERATOR + ERRORS_ROOT;
+ public static final String BLOBSTORE_SUBTREE = ZK_SEPERATOR + BLOBSTORE_ROOT;
+ public static final String BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE = ZK_SEPERATOR + BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_ROOT;
+ public static final String NIMBUSES_SUBTREE = ZK_SEPERATOR + NIMBUSES_ROOT;
+ public static final String CREDENTIALS_SUBTREE = ZK_SEPERATOR + CREDENTIALS_ROOT;
+ public static final String LOGCONFIG_SUBTREE = ZK_SEPERATOR + LOGCONFIG_ROOT;
+ public static final String PROFILERCONFIG_SUBTREE = ZK_SEPERATOR + PROFILERCONFIG_ROOT;
+
+ // A singleton instance allows us to mock delegated static methods in our
+ // tests by subclassing.
+ private static final ClusterUtils INSTANCE = new ClusterUtils();
+ private static ClusterUtils _instance = INSTANCE;
+
+ /**
+ * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that overrides the
+ * implementation of the delegated method.
+ *
+ * @param u a Cluster instance
+ */
+ public static void setInstance(ClusterUtils u) {
+ _instance = u;
+ }
+
+ /**
+ * Resets the singleton instance to the default. This is helpful to reset the class to its original functionality when mocking is no longer desired.
+ */
+ public static void resetInstance() {
+ _instance = INSTANCE;
+ }
+
+ public static List<ACL> mkTopoOnlyAcls(Map topoConf) throws NoSuchAlgorithmException {
+ List<ACL> aclList = null;
+ String payload = (String) topoConf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD);
+ if (Utils.isZkAuthenticationConfiguredTopology(topoConf)) {
+ aclList = new ArrayList<>();
+ ACL acl1 = ZooDefs.Ids.CREATOR_ALL_ACL.get(0);
+ aclList.add(acl1);
+ ACL acl2 = new ACL(ZooDefs.Perms.READ, new Id("digest", DigestAuthenticationProvider.generateDigest(payload)));
+ aclList.add(acl2);
+ }
+ return aclList;
+ }
+
+ public static String supervisorPath(String id) {
+ return SUPERVISORS_SUBTREE + ZK_SEPERATOR + id;
+ }
+
+ public static String assignmentPath(String id) {
+ return ASSIGNMENTS_SUBTREE + ZK_SEPERATOR + id;
+ }
+
+ public static String blobstorePath(String key) {
+ return BLOBSTORE_SUBTREE + ZK_SEPERATOR + key;
+ }
+
+ public static String blobstoreMaxKeySequenceNumberPath(String key) {
+ return BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE + ZK_SEPERATOR + key;
+ }
+
+ public static String nimbusPath(String id) {
+ return NIMBUSES_SUBTREE + ZK_SEPERATOR + id;
+ }
+
+ public static String stormPath(String id) {
+ return STORMS_SUBTREE + ZK_SEPERATOR + id;
+ }
+
+ public static String workerbeatStormRoot(String stormId) {
+ return WORKERBEATS_SUBTREE + ZK_SEPERATOR + stormId;
+ }
+
+ public static String workerbeatPath(String stormId, String node, Long port) {
+ return workerbeatStormRoot(stormId) + ZK_SEPERATOR + node + "-" + port;
+ }
+
+ public static String backpressureStormRoot(String stormId) {
+ return BACKPRESSURE_SUBTREE + ZK_SEPERATOR + stormId;
+ }
+
+ public static String backpressurePath(String stormId, String node, Long port) {
+ return backpressureStormRoot(stormId) + ZK_SEPERATOR + node + "-" + port;
+ }
+
+ public static String errorStormRoot(String stormId) {
+ return ERRORS_SUBTREE + ZK_SEPERATOR + stormId;
+ }
+
+ public static String errorPath(String stormId, String componentId) {
+ try {
+ return errorStormRoot(stormId) + ZK_SEPERATOR + URLEncoder.encode(componentId, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw Utils.wrapInRuntime(e);
+ }
+ }
+
+ public static String lastErrorPath(String stormId, String componentId) {
+ return errorPath(stormId, componentId) + "-last-error";
+ }
+
+ public static String credentialsPath(String stormId) {
+ return CREDENTIALS_SUBTREE + ZK_SEPERATOR + stormId;
+ }
+
+ public static String logConfigPath(String stormId) {
+ return LOGCONFIG_SUBTREE + ZK_SEPERATOR + stormId;
+ }
+
+ public static String profilerConfigPath(String stormId) {
+ return PROFILERCONFIG_SUBTREE + ZK_SEPERATOR + stormId;
+ }
+
+ public static String profilerConfigPath(String stormId, String host, Long port, ProfileAction requestType) {
+ return profilerConfigPath(stormId) + ZK_SEPERATOR + host + "_" + port + "_" + requestType;
+ }
+
+ public static <T> T maybeDeserialize(byte[] serialized, Class<T> clazz) {
+ if (serialized != null) {
+ return Utils.deserialize(serialized, clazz);
+ }
+ return null;
+ }
+
+ /**
+ * Ensures that we only return heartbeats for executors assigned to this worker
+ * @param executors
+ * @param workerHeartbeat
+ * @return
+ */
+ public static Map<ExecutorInfo, ExecutorBeat> convertExecutorBeats(List<ExecutorInfo> executors, ClusterWorkerHeartbeat workerHeartbeat) {
+ Map<ExecutorInfo, ExecutorBeat> executorWhb = new HashMap<>();
+ Map<ExecutorInfo, ExecutorStats> executorStatsMap = workerHeartbeat.get_executor_stats();
+ for (ExecutorInfo executor : executors) {
+ if (executorStatsMap.containsKey(executor)) {
+ int time = workerHeartbeat.get_time_secs();
+ int uptime = workerHeartbeat.get_uptime_secs();
+ ExecutorStats executorStats = workerHeartbeat.get_executor_stats().get(executor);
+ ExecutorBeat executorBeat = new ExecutorBeat(time, uptime, executorStats);
+ executorWhb.put(executor, executorBeat);
+ }
+ }
+ return executorWhb;
+ }
+
+ public IStormClusterState mkStormClusterStateImpl(Object stateStorage, List<ACL> acls, ClusterStateContext context) throws Exception {
+ if (stateStorage instanceof IStateStorage) {
+ return new StormClusterStateImpl((IStateStorage) stateStorage, acls, context, false);
+ } else {
+ IStateStorage Storage = _instance.mkStateStorageImpl((Map) stateStorage, (Map) stateStorage, acls, context);
+ return new StormClusterStateImpl(Storage, acls, context, true);
+ }
+ }
+
+ public IStateStorage mkStateStorageImpl(Map config, Map auth_conf, List<ACL> acls, ClusterStateContext context) throws Exception {
+ String className = null;
+ IStateStorage stateStorage = null;
+ if (config.get(Config.STORM_CLUSTER_STATE_STORE) != null) {
+ className = (String) config.get(Config.STORM_CLUSTER_STATE_STORE);
+ } else {
+ className = "org.apache.storm.cluster.ZKStateStorageFactory";
+ }
+ Class clazz = Class.forName(className);
+ StateStorageFactory storageFactory = (StateStorageFactory) clazz.newInstance();
+ stateStorage = storageFactory.mkStore(config, auth_conf, acls, context);
+ return stateStorage;
+ }
+
+ public static IStateStorage mkStateStorage(Map config, Map auth_conf, List<ACL> acls, ClusterStateContext context) throws Exception {
+ return _instance.mkStateStorageImpl(config, auth_conf, acls, context);
+ }
+
+ public static IStormClusterState mkStormClusterState(Object StateStorage, List<ACL> acls, ClusterStateContext context) throws Exception {
+ return _instance.mkStormClusterStateImpl(StateStorage, acls, context);
+ }
+
+ public static String stringifyError(Throwable error) {
+ StringWriter result = new StringWriter();
+ PrintWriter printWriter = new PrintWriter(result);
+ error.printStackTrace(printWriter);
+ return result.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/ExecutorBeat.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/ExecutorBeat.java b/storm-core/src/jvm/org/apache/storm/cluster/ExecutorBeat.java
new file mode 100644
index 0000000..b32615e
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/ExecutorBeat.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import org.apache.storm.generated.ExecutorStats;
+
+public class ExecutorBeat {
+ private final int timeSecs;
+ private final int uptime;
+ private final ExecutorStats stats;
+
+ public ExecutorBeat(int timeSecs, int uptime, ExecutorStats stats) {
+ this.timeSecs = timeSecs;
+ this.uptime = uptime;
+ this.stats = stats;
+ }
+
+ public int getTimeSecs() {
+ return timeSecs;
+ }
+
+ public int getUptime() {
+ return uptime;
+ }
+
+ public ExecutorStats getStats() {
+ return stats;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/IStateStorage.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/IStateStorage.java b/storm-core/src/jvm/org/apache/storm/cluster/IStateStorage.java
new file mode 100644
index 0000000..aa731ff
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/IStateStorage.java
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.curator.framework.state.ConnectionStateListener;
+import org.apache.storm.callback.ZKStateChangedCallback;
+import org.apache.zookeeper.data.ACL;
+
+/**
+ * StateStorage provides the API for the pluggable state store used by the
+ * Storm daemons. Data is stored in path/value format, and the store supports
+ * listing sub-paths at a given path.
+ * All data should be available across all nodes with eventual consistency.
+ *
+ * IMPORTANT NOTE:
+ * Heartbeats have different api calls used to interact with them. The root
+ * path (/) may or may not be the same as the root path for the other api calls.
+ *
+ * For example, performing these two calls:
+ * set_data("/path", data, acls);
+ * void set_worker_hb("/path", heartbeat, acls);
+ * may or may not cause a collision in "/path".
+ * Never use the same paths with the *_hb* methods as you do with the others.
+ */
+public interface IStateStorage {
+
+ /**
+ * Registers a callback function that gets called when CuratorEvents happen.
+ * @param callback is a clojure IFn that accepts the type - translated to
+ * clojure keyword as in zookeeper - and the path: (callback type path)
+ * @return is an id that can be passed to unregister(...) to unregister the
+ * callback.
+ */
+ String register(ZKStateChangedCallback callback);
+
+ /**
+ * Unregisters a callback function that was registered with register(...).
+ * @param id is the String id that was returned from register(...).
+ */
+ void unregister(String id);
+
+ /**
+ * Path will be appended with a monotonically increasing integer, a new node
+ * will be created there, and data will be put at that node.
+ * @param path The path that the monotonically increasing integer suffix will
+ * be added to.
+ * @param data The data that will be written at the suffixed path's node.
+ * @param acls The acls to apply to the path. May be null.
+ * @return The path with the integer suffix appended.
+ */
+ String create_sequential(String path, byte[] data, List<ACL> acls);
+
+ /**
+ * Creates nodes for path and all its parents. Path elements are separated by
+ * a "/", as in *nix filesystem notation. Equivalent to mkdir -p in *nix.
+ * @param path The path to create, along with all its parents.
+ * @param acls The acls to apply to the path. May be null.
+ * @return path
+ */
+ void mkdirs(String path, List<ACL> acls);
+
+ /**
+ * Deletes the node at a given path, and any child nodes that may exist.
+ * @param path The path to delete
+ */
+ void delete_node(String path);
+
+ /**
+ * Creates an ephemeral node at path. Ephemeral nodes are destroyed
+ * by the store when the client disconnects.
+ * @param path The path where a node will be created.
+ * @param data The data to be written at the node.
+ * @param acls The acls to apply to the path. May be null.
+ */
+ void set_ephemeral_node(String path, byte[] data, List<ACL> acls);
+
+ /**
+ * Gets the 'version' of the node at a path. Optionally sets a watch
+ * on that node. The version should increase whenever a write happens.
+ * @param path The path to get the version of.
+ * @param watch Whether or not to set a watch on the path. Watched paths
+ * emit events which are consumed by functions registered with the
+ * register method. Very useful for catching updates to nodes.
+ * @return The integer version of this node.
+ */
+ Integer get_version(String path, boolean watch) throws Exception;
+
+ /**
+ * Check if a node exists and optionally set a watch on the path.
+ * @param path The path to check for the existence of a node.
+ * @param watch Whether or not to set a watch on the path. Watched paths
+ * emit events which are consumed by functions registered with the
+ * register method. Very useful for catching updates to nodes.
+ * @return Whether or not a node exists at path.
+ */
+ boolean node_exists(String path, boolean watch);
+
+ /**
+ * Get a list of paths of all the child nodes which exist immediately
+ * under path.
+ * @param path The path to look under
+ * @param watch Whether or not to set a watch on the path. Watched paths
+ * emit events which are consumed by functions registered with the
+ * register method. Very useful for catching updates to nodes.
+ * @return list of string paths under path.
+ */
+ List<String> get_children(String path, boolean watch);
+
+ /**
+ * Close the connection to the data store.
+ */
+ void close();
+
+ /**
+ * Set the value of the node at path to data.
+ * @param path The path whose node we want to set.
+ * @param data The data to put in the node.
+ * @param acls The acls to apply to the path. May be null.
+ */
+ void set_data(String path, byte[] data, List<ACL> acls);
+
+ /**
+ * Get the data from the node at path
+ * @param path The path to look under
+ * @param watch Whether or not to set a watch on the path. Watched paths
+ * emit events which are consumed by functions registered with the
+ * register method. Very useful for catching updates to nodes.
+ * @return The data at the node.
+ */
+ byte[] get_data(String path, boolean watch);
+
+ /**
+ * Get the data at the node along with its version. Data is returned
+ * in an Map with the keys data and version.
+ * @param path The path to look under
+ * @param watch Whether or not to set a watch on the path. Watched paths
+ * emit events which are consumed by functions registered with the
+ * register method. Very useful for catching updates to nodes.
+ * @return the data with a version
+ */
+ VersionedData<byte[]> get_data_with_version(String path, boolean watch);
+
+ /**
+ * Write a worker heartbeat at the path.
+ * @param path The path whose node we want to set.
+ * @param data The data to put in the node.
+ * @param acls The acls to apply to the path. May be null.
+ */
+ void set_worker_hb(String path, byte[] data, List<ACL> acls);
+
+ /**
+ * Get the heartbeat from the node at path
+ * @param path The path to look under
+ * @param watch Whether or not to set a watch on the path. Watched paths
+ * emit events which are consumed by functions registered with the
+ * register method. Very useful for catching updates to nodes.
+ * @return The heartbeat at the node.
+ */
+ byte[] get_worker_hb(String path, boolean watch);
+
+ /**
+ * Get a list of paths of all the child nodes which exist immediately
+ * under path. This is similar to get_children, but must be used for
+ * any nodes
+ * @param path The path to look under
+ * @param watch Whether or not to set a watch on the path. Watched paths
+ * emit events which are consumed by functions registered with the
+ * register method. Very useful for catching updates to nodes.
+ * @return list of string paths under path.
+ */
+ List<String> get_worker_hb_children(String path, boolean watch);
+
+ /**
+ * Deletes the heartbeat at a given path, and any child nodes that may exist.
+ * @param path The path to delete.
+ */
+ void delete_worker_hb(String path);
+
+ /**
+ * Add a StateStorageListener to the connection.
+ * @param listener A StateStorageListener to handle changing cluster state
+ * events.
+ */
+ void add_listener(final ConnectionStateListener listener);
+
+ /**
+ * Force consistency on a path. Any writes committed on the path before
+ * this call will be completely propagated when it returns.
+ * @param path The path to synchronize.
+ */
+ void sync_path(String path);
+
+ /**
+ * Allows us to delete the znodes within /storm/blobstore/key_name
+ * whose znodes start with the corresponding nimbusHostPortInfo
+ * @param path /storm/blobstore/key_name
+ * @param nimbusHostPortInfo Contains the host port information of
+ * a nimbus node.
+ */
+ void delete_node_blobstore(String path, String nimbusHostPortInfo);
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java b/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java
new file mode 100644
index 0000000..a6f07ed
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import org.apache.storm.generated.*;
+import org.apache.storm.nimbus.NimbusInfo;
+
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.Map;
+
+public interface IStormClusterState {
+ public List<String> assignments(Runnable callback);
+
+ public Assignment assignmentInfo(String stormId, Runnable callback);
+
+ public VersionedData<Assignment> assignmentInfoWithVersion(String stormId, Runnable callback);
+
+ public Integer assignmentVersion(String stormId, Runnable callback) throws Exception;
+
+ public List<String> blobstoreInfo(String blobKey);
+
+ public List nimbuses();
+
+ public void addNimbusHost(String nimbusId, NimbusSummary nimbusSummary);
+
+ public List<String> activeStorms();
+
+ public StormBase stormBase(String stormId, Runnable callback);
+
+ public ClusterWorkerHeartbeat getWorkerHeartbeat(String stormId, String node, Long port);
+
+ public List<ProfileRequest> getWorkerProfileRequests(String stormId, NodeInfo nodeInfo);
+
+ public List<ProfileRequest> getTopologyProfileRequests(String stormId);
+
+ public void setWorkerProfileRequest(String stormId, ProfileRequest profileRequest);
+
+ public void deleteTopologyProfileRequests(String stormId, ProfileRequest profileRequest);
+
+ public Map<ExecutorInfo, ExecutorBeat> executorBeats(String stormId, Map<List<Long>, NodeInfo> executorNodePort);
+
+ public List<String> supervisors(Runnable callback);
+
+ public SupervisorInfo supervisorInfo(String supervisorId); // returns nil if doesn't exist
+
+ public void setupHeatbeats(String stormId);
+
+ public void teardownHeartbeats(String stormId);
+
+ public void teardownTopologyErrors(String stormId);
+
+ public List<String> heartbeatStorms();
+
+ public List<String> errorTopologies();
+
+ public List<String> backpressureTopologies();
+
+ public void setTopologyLogConfig(String stormId, LogConfig logConfig);
+
+ public LogConfig topologyLogConfig(String stormId, Runnable cb);
+
+ public void workerHeartbeat(String stormId, String node, Long port, ClusterWorkerHeartbeat info);
+
+ public void removeWorkerHeartbeat(String stormId, String node, Long port);
+
+ public void supervisorHeartbeat(String supervisorId, SupervisorInfo info);
+
+ public void workerBackpressure(String stormId, String node, Long port, boolean on);
+
+ public boolean topologyBackpressure(String stormId, Runnable callback);
+
+ public void setupBackpressure(String stormId);
+
+ public void removeBackpressure(String stormId);
+
+ public void removeWorkerBackpressure(String stormId, String node, Long port);
+
+ public void activateStorm(String stormId, StormBase stormBase);
+
+ public void updateStorm(String stormId, StormBase newElems);
+
+ public void removeStormBase(String stormId);
+
+ public void setAssignment(String stormId, Assignment info);
+
+ public void setupBlobstore(String key, NimbusInfo nimbusInfo, Integer versionInfo);
+
+ public List<String> activeKeys();
+
+ public List<String> blobstore(Runnable callback);
+
+ public void removeStorm(String stormId);
+
+ public void removeBlobstoreKey(String blobKey);
+
+ public void removeKeyVersion(String blobKey);
+
+ public void reportError(String stormId, String componentId, String node, Long port, Throwable error);
+
+ public List<ErrorInfo> errors(String stormId, String componentId);
+
+ public ErrorInfo lastError(String stormId, String componentId);
+
+ public void setCredentials(String stormId, Credentials creds, Map topoConf) throws NoSuchAlgorithmException;
+
+ public Credentials credentials(String stormId, Runnable callback);
+
+ public void disconnect();
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/StateStorageFactory.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/StateStorageFactory.java b/storm-core/src/jvm/org/apache/storm/cluster/StateStorageFactory.java
new file mode 100644
index 0000000..0929750
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/StateStorageFactory.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.zookeeper.data.ACL;
+
+public interface StateStorageFactory {
+
+ IStateStorage mkStore(Map config, Map auth_conf, List<ACL> acls, ClusterStateContext context);
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java b/storm-core/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java
new file mode 100644
index 0000000..972d778
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java
@@ -0,0 +1,720 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.state.*;
+import org.apache.curator.framework.state.ConnectionState;
+import org.apache.storm.callback.ZKStateChangedCallback;
+import org.apache.storm.generated.*;
+import org.apache.storm.nimbus.NimbusInfo;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.apache.storm.zookeeper.Zookeeper;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.ACL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.NoSuchAlgorithmException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class StormClusterStateImpl implements IStormClusterState {
+
+ private static Logger LOG = LoggerFactory.getLogger(StormClusterStateImpl.class);
+
+ private IStateStorage stateStorage;
+
+ private ConcurrentHashMap<String, Runnable> assignmentInfoCallback;
+ private ConcurrentHashMap<String, Runnable> assignmentInfoWithVersionCallback;
+ private ConcurrentHashMap<String, Runnable> assignmentVersionCallback;
+ private AtomicReference<Runnable> supervisorsCallback;
+ // we want to reigister a topo directory getChildren callback for all workers of this dir
+ private ConcurrentHashMap<String, Runnable> backPressureCallback;
+ private AtomicReference<Runnable> assignmentsCallback;
+ private ConcurrentHashMap<String, Runnable> stormBaseCallback;
+ private AtomicReference<Runnable> blobstoreCallback;
+ private ConcurrentHashMap<String, Runnable> credentialsCallback;
+ private ConcurrentHashMap<String, Runnable> logConfigCallback;
+
+ private List<ACL> acls;
+ private String stateId;
+ private boolean solo;
+
+ public StormClusterStateImpl(IStateStorage StateStorage, List<ACL> acls, ClusterStateContext context, boolean solo) throws Exception {
+
+ this.stateStorage = StateStorage;
+ this.solo = solo;
+ this.acls = acls;
+
+ assignmentInfoCallback = new ConcurrentHashMap<>();
+ assignmentInfoWithVersionCallback = new ConcurrentHashMap<>();
+ assignmentVersionCallback = new ConcurrentHashMap<>();
+ supervisorsCallback = new AtomicReference<>();
+ backPressureCallback = new ConcurrentHashMap<>();
+ assignmentsCallback = new AtomicReference<>();
+ stormBaseCallback = new ConcurrentHashMap<>();
+ credentialsCallback = new ConcurrentHashMap<>();
+ logConfigCallback = new ConcurrentHashMap<>();
+ blobstoreCallback = new AtomicReference<>();
+
+ stateId = this.stateStorage.register(new ZKStateChangedCallback() {
+
+ public void changed(Watcher.Event.EventType type, String path) {
+ List<String> toks = Zookeeper.tokenizePath(path);
+ int size = toks.size();
+ if (size >= 1) {
+ String root = toks.get(0);
+ if (root.equals(ClusterUtils.ASSIGNMENTS_ROOT)) {
+ if (size == 1) {
+ // set null and get the old value
+ issueCallback(assignmentsCallback);
+ } else {
+ issueMapCallback(assignmentInfoCallback, toks.get(1));
+ issueMapCallback(assignmentVersionCallback, toks.get(1));
+ issueMapCallback(assignmentInfoWithVersionCallback, toks.get(1));
+ }
+
+ } else if (root.equals(ClusterUtils.SUPERVISORS_ROOT)) {
+ issueCallback(supervisorsCallback);
+ } else if (root.equals(ClusterUtils.BLOBSTORE_ROOT)) {
+ issueCallback(blobstoreCallback);
+ } else if (root.equals(ClusterUtils.STORMS_ROOT) && size > 1) {
+ issueMapCallback(stormBaseCallback, toks.get(1));
+ } else if (root.equals(ClusterUtils.CREDENTIALS_ROOT) && size > 1) {
+ issueMapCallback(credentialsCallback, toks.get(1));
+ } else if (root.equals(ClusterUtils.LOGCONFIG_ROOT) && size > 1) {
+ issueMapCallback(logConfigCallback, toks.get(1));
+ } else if (root.equals(ClusterUtils.BACKPRESSURE_ROOT) && size > 1) {
+ issueMapCallback(backPressureCallback, toks.get(1));
+ } else {
+ LOG.error("{} Unknown callback for subtree {}", new RuntimeException("Unknown callback for this path"), path);
+ Runtime.getRuntime().exit(30);
+ }
+
+ }
+
+ return;
+ }
+
+ });
+
+ String[] pathlist = { ClusterUtils.ASSIGNMENTS_SUBTREE,
+ ClusterUtils.STORMS_SUBTREE,
+ ClusterUtils.SUPERVISORS_SUBTREE,
+ ClusterUtils.WORKERBEATS_SUBTREE,
+ ClusterUtils.ERRORS_SUBTREE,
+ ClusterUtils.BLOBSTORE_SUBTREE,
+ ClusterUtils.NIMBUSES_SUBTREE,
+ ClusterUtils.LOGCONFIG_SUBTREE,
+ ClusterUtils.BACKPRESSURE_SUBTREE };
+ for (String path : pathlist) {
+ this.stateStorage.mkdirs(path, acls);
+ }
+
+ }
+
+ protected void issueCallback(AtomicReference<Runnable> cb) {
+ Runnable callback = cb.getAndSet(null);
+ if (callback != null)
+ callback.run();
+ }
+
+ protected void issueMapCallback(ConcurrentHashMap<String, Runnable> callbackConcurrentHashMap, String key) {
+ Runnable callback = callbackConcurrentHashMap.remove(key);
+ if (callback != null)
+ callback.run();
+ }
+
+ @Override
+ public List<String> assignments(Runnable callback) {
+ if (callback != null) {
+ assignmentsCallback.set(callback);
+ }
+ return stateStorage.get_children(ClusterUtils.ASSIGNMENTS_SUBTREE, callback != null);
+ }
+
+ @Override
+ public Assignment assignmentInfo(String stormId, Runnable callback) {
+ if (callback != null) {
+ assignmentInfoCallback.put(stormId, callback);
+ }
+ byte[] serialized = stateStorage.get_data(ClusterUtils.assignmentPath(stormId), callback != null);
+ return ClusterUtils.maybeDeserialize(serialized, Assignment.class);
+ }
+
+ @Override
+ public VersionedData<Assignment> assignmentInfoWithVersion(String stormId, Runnable callback) {
+ if (callback != null) {
+ assignmentInfoWithVersionCallback.put(stormId, callback);
+ }
+ Assignment assignment = null;
+ Integer version = 0;
+ VersionedData<byte[]> dataWithVersion = stateStorage.get_data_with_version(ClusterUtils.assignmentPath(stormId), callback != null);
+ if (dataWithVersion != null) {
+ assignment = ClusterUtils.maybeDeserialize(dataWithVersion.getData(), Assignment.class);
+ version = dataWithVersion.getVersion();
+ }
+ return new VersionedData<Assignment>(version, assignment);
+ }
+
+ @Override
+ public Integer assignmentVersion(String stormId, Runnable callback) throws Exception {
+ if (callback != null) {
+ assignmentVersionCallback.put(stormId, callback);
+ }
+ return stateStorage.get_version(ClusterUtils.assignmentPath(stormId), callback != null);
+ }
+
+ // blobstore state
+ @Override
+ public List<String> blobstoreInfo(String blobKey) {
+ String path = ClusterUtils.blobstorePath(blobKey);
+ stateStorage.sync_path(path);
+ return stateStorage.get_children(path, false);
+ }
+
+ @Override
+ public List nimbuses() {
+ List<NimbusSummary> nimbusSummaries = new ArrayList<>();
+ List<String> nimbusIds = stateStorage.get_children(ClusterUtils.NIMBUSES_SUBTREE, false);
+ for (String nimbusId : nimbusIds) {
+ byte[] serialized = stateStorage.get_data(ClusterUtils.nimbusPath(nimbusId), false);
+ NimbusSummary nimbusSummary = ClusterUtils.maybeDeserialize(serialized, NimbusSummary.class);
+ nimbusSummaries.add(nimbusSummary);
+ }
+ return nimbusSummaries;
+ }
+
+ @Override
+ public void addNimbusHost(final String nimbusId, final NimbusSummary nimbusSummary) {
+ // explicit delete for ephmeral node to ensure this session creates the entry.
+ stateStorage.delete_node(ClusterUtils.nimbusPath(nimbusId));
+ stateStorage.add_listener(new ConnectionStateListener() {
+ @Override
+ public void stateChanged(CuratorFramework curatorFramework, ConnectionState connectionState) {
+ LOG.info("Connection state listener invoked, zookeeper connection state has changed to {}", connectionState);
+ if (connectionState.equals(ConnectionState.RECONNECTED)) {
+ LOG.info("Connection state has changed to reconnected so setting nimbuses entry one more time");
+ // explicit delete for ephmeral node to ensure this session creates the entry.
+ stateStorage.delete_node(ClusterUtils.nimbusPath(nimbusId));
+ stateStorage.set_ephemeral_node(ClusterUtils.nimbusPath(nimbusId), Utils.serialize(nimbusSummary), acls);
+ }
+
+ }
+ });
+
+ stateStorage.set_ephemeral_node(ClusterUtils.nimbusPath(nimbusId), Utils.serialize(nimbusSummary), acls);
+ }
+
+ @Override
+ public List<String> activeStorms() {
+ return stateStorage.get_children(ClusterUtils.STORMS_SUBTREE, false);
+ }
+
+ @Override
+ public StormBase stormBase(String stormId, Runnable callback) {
+ if (callback != null) {
+ stormBaseCallback.put(stormId, callback);
+ }
+ return ClusterUtils.maybeDeserialize(stateStorage.get_data(ClusterUtils.stormPath(stormId), callback != null), StormBase.class);
+ }
+
+ @Override
+ public ClusterWorkerHeartbeat getWorkerHeartbeat(String stormId, String node, Long port) {
+ byte[] bytes = stateStorage.get_worker_hb(ClusterUtils.workerbeatPath(stormId, node, port), false);
+ return ClusterUtils.maybeDeserialize(bytes, ClusterWorkerHeartbeat.class);
+
+ }
+
+ @Override
+ public List<ProfileRequest> getWorkerProfileRequests(String stormId, NodeInfo nodeInfo) {
+ List<ProfileRequest> requests = new ArrayList<>();
+ List<ProfileRequest> profileRequests = getTopologyProfileRequests(stormId);
+ for (ProfileRequest profileRequest : profileRequests) {
+ NodeInfo nodeInfo1 = profileRequest.get_nodeInfo();
+ if (nodeInfo1.equals(nodeInfo))
+ requests.add(profileRequest);
+ }
+ return requests;
+ }
+
+ @Override
+ public List<ProfileRequest> getTopologyProfileRequests(String stormId) {
+ List<ProfileRequest> profileRequests = new ArrayList<>();
+ String path = ClusterUtils.profilerConfigPath(stormId);
+ if (stateStorage.node_exists(path, false)) {
+ List<String> strs = stateStorage.get_children(path, false);
+ for (String str : strs) {
+ String childPath = path + ClusterUtils.ZK_SEPERATOR + str;
+ byte[] raw = stateStorage.get_data(childPath, false);
+ ProfileRequest request = ClusterUtils.maybeDeserialize(raw, ProfileRequest.class);
+ if (request != null)
+ profileRequests.add(request);
+ }
+ }
+ return profileRequests;
+ }
+
+ @Override
+ public void setWorkerProfileRequest(String stormId, ProfileRequest profileRequest) {
+ ProfileAction profileAction = profileRequest.get_action();
+ String host = profileRequest.get_nodeInfo().get_node();
+ Long port = profileRequest.get_nodeInfo().get_port_iterator().next();
+ String path = ClusterUtils.profilerConfigPath(stormId, host, port, profileAction);
+ stateStorage.set_data(path, Utils.serialize(profileRequest), acls);
+ }
+
+ @Override
+ public void deleteTopologyProfileRequests(String stormId, ProfileRequest profileRequest) {
+ ProfileAction profileAction = profileRequest.get_action();
+ String host = profileRequest.get_nodeInfo().get_node();
+ Long port = profileRequest.get_nodeInfo().get_port_iterator().next();
+ String path = ClusterUtils.profilerConfigPath(stormId, host, port, profileAction);
+ stateStorage.delete_node(path);
+ }
+
+ /**
+ * need to take executor->node+port in explicitly so that we don't run into a situation where a long dead worker with a skewed clock overrides all the
+ * timestamps. By only checking heartbeats with an assigned node+port, and only reading executors from that heartbeat that are actually assigned, we avoid
+ * situations like that
+ *
+ * @param stormId
+ * @param executorNodePort
+ * @return
+ */
+ @Override
+ public Map<ExecutorInfo, ExecutorBeat> executorBeats(String stormId, Map<List<Long>, NodeInfo> executorNodePort) {
+ Map<ExecutorInfo, ExecutorBeat> executorWhbs = new HashMap<>();
+
+ Map<NodeInfo, List<List<Long>>> nodePortExecutors = Utils.reverseMap(executorNodePort);
+
+ for (Map.Entry<NodeInfo, List<List<Long>>> entry : nodePortExecutors.entrySet()) {
+
+ String node = entry.getKey().get_node();
+ Long port = entry.getKey().get_port_iterator().next();
+ ClusterWorkerHeartbeat whb = getWorkerHeartbeat(stormId, node, port);
+ List<ExecutorInfo> executorInfoList = new ArrayList<>();
+ for (List<Long> list : entry.getValue()) {
+ executorInfoList.add(new ExecutorInfo(list.get(0).intValue(), list.get(list.size() - 1).intValue()));
+ }
+ if (whb != null)
+ executorWhbs.putAll(ClusterUtils.convertExecutorBeats(executorInfoList, whb));
+ }
+ return executorWhbs;
+ }
+
+ @Override
+ public List<String> supervisors(Runnable callback) {
+ if (callback != null) {
+ supervisorsCallback.set(callback);
+ }
+ return stateStorage.get_children(ClusterUtils.SUPERVISORS_SUBTREE, callback != null);
+ }
+
+ @Override
+ public SupervisorInfo supervisorInfo(String supervisorId) {
+ String path = ClusterUtils.supervisorPath(supervisorId);
+ return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), SupervisorInfo.class);
+ }
+
+ @Override
+ public void setupHeatbeats(String stormId) {
+ stateStorage.mkdirs(ClusterUtils.workerbeatStormRoot(stormId), acls);
+ }
+
+ @Override
+ public void teardownHeartbeats(String stormId) {
+ try {
+ stateStorage.delete_worker_hb(ClusterUtils.workerbeatStormRoot(stormId));
+ } catch (Exception e) {
+ if (Utils.exceptionCauseIsInstanceOf(KeeperException.class, e)) {
+ // do nothing
+ LOG.warn("Could not teardown heartbeats for {}.", stormId);
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ @Override
+ public void teardownTopologyErrors(String stormId) {
+ try {
+ stateStorage.delete_node(ClusterUtils.errorStormRoot(stormId));
+ } catch (Exception e) {
+ if (Utils.exceptionCauseIsInstanceOf(KeeperException.class, e)) {
+ // do nothing
+ LOG.warn("Could not teardown errors for {}.", stormId);
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ @Override
+ public List<String> heartbeatStorms() {
+ return stateStorage.get_worker_hb_children(ClusterUtils.WORKERBEATS_SUBTREE, false);
+ }
+
+ @Override
+ public List<String> errorTopologies() {
+ return stateStorage.get_children(ClusterUtils.ERRORS_SUBTREE, false);
+ }
+
+ @Override
+ public List<String> backpressureTopologies() {
+ return stateStorage.get_children(ClusterUtils.BACKPRESSURE_SUBTREE, false);
+ }
+
+ @Override
+ public void setTopologyLogConfig(String stormId, LogConfig logConfig) {
+ stateStorage.set_data(ClusterUtils.logConfigPath(stormId), Utils.serialize(logConfig), acls);
+ }
+
+ @Override
+ public LogConfig topologyLogConfig(String stormId, Runnable cb) {
+ if (cb != null){
+ logConfigCallback.put(stormId, cb);
+ }
+ String path = ClusterUtils.logConfigPath(stormId);
+ return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, cb != null), LogConfig.class);
+ }
+
+ @Override
+ public void workerHeartbeat(String stormId, String node, Long port, ClusterWorkerHeartbeat info) {
+ if (info != null) {
+ String path = ClusterUtils.workerbeatPath(stormId, node, port);
+ stateStorage.set_worker_hb(path, Utils.serialize(info), acls);
+ }
+ }
+
+ @Override
+ public void removeWorkerHeartbeat(String stormId, String node, Long port) {
+ String path = ClusterUtils.workerbeatPath(stormId, node, port);
+ stateStorage.delete_worker_hb(path);
+ }
+
+ @Override
+ public void supervisorHeartbeat(String supervisorId, SupervisorInfo info) {
+ String path = ClusterUtils.supervisorPath(supervisorId);
+ stateStorage.set_ephemeral_node(path, Utils.serialize(info), acls);
+ }
+
+ /**
+ * if znode exists and to be not on?, delete; if exists and on?, do nothing; if not exists and to be on?, create; if not exists and not on?, do nothing;
+ *
+ * @param stormId
+ * @param node
+ * @param port
+ * @param on
+ */
+ @Override
+ public void workerBackpressure(String stormId, String node, Long port, boolean on) {
+ String path = ClusterUtils.backpressurePath(stormId, node, port);
+ boolean existed = stateStorage.node_exists(path, false);
+ if (existed) {
+ if (on == false)
+ stateStorage.delete_node(path);
+
+ } else {
+ if (on == true) {
+ stateStorage.set_ephemeral_node(path, null, acls);
+ }
+ }
+ }
+
+ /**
+ * Check whether a topology is in throttle-on status or not:
+ * if the backpresure/storm-id dir is not empty, this topology has throttle-on, otherwise throttle-off.
+ *
+ * @param stormId
+ * @param callback
+ * @return
+ */
+ @Override
+ public boolean topologyBackpressure(String stormId, Runnable callback) {
+ if (callback != null) {
+ backPressureCallback.put(stormId, callback);
+ }
+ String path = ClusterUtils.backpressureStormRoot(stormId);
+ List<String> childrens = null;
+ if(stateStorage.node_exists(path, false)) {
+ childrens = stateStorage.get_children(path, callback != null);
+ } else {
+ childrens = new ArrayList<>();
+ }
+ return childrens.size() > 0;
+
+ }
+
+ @Override
+ public void setupBackpressure(String stormId) {
+ stateStorage.mkdirs(ClusterUtils.backpressureStormRoot(stormId), acls);
+ }
+
+ @Override
+ public void removeBackpressure(String stormId) {
+ try {
+ stateStorage.delete_node(ClusterUtils.backpressureStormRoot(stormId));
+ } catch (Exception e) {
+ if (Utils.exceptionCauseIsInstanceOf(KeeperException.class, e)) {
+ // do nothing
+ LOG.warn("Could not teardown backpressure node for {}.", stormId);
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ @Override
+ public void removeWorkerBackpressure(String stormId, String node, Long port) {
+ String path = ClusterUtils.backpressurePath(stormId, node, port);
+ boolean existed = stateStorage.node_exists(path, false);
+ if (existed) {
+ stateStorage.delete_node(path);
+ }
+ }
+
+ @Override
+ public void activateStorm(String stormId, StormBase stormBase) {
+ String path = ClusterUtils.stormPath(stormId);
+ stateStorage.set_data(path, Utils.serialize(stormBase), acls);
+ }
+
+ /**
+ * To update this function due to APersistentMap/APersistentSet is clojure's structure
+ *
+ * @param stormId
+ * @param newElems
+ */
+ @Override
+ public void updateStorm(String stormId, StormBase newElems) {
+
+ StormBase stormBase = stormBase(stormId, null);
+ if (stormBase.get_component_executors() != null) {
+
+ Map<String, Integer> newComponentExecutors = new HashMap<>();
+ Map<String, Integer> componentExecutors = newElems.get_component_executors();
+ // componentExecutors maybe be APersistentMap, which don't support "put"
+ for (Map.Entry<String, Integer> entry : componentExecutors.entrySet()) {
+ newComponentExecutors.put(entry.getKey(), entry.getValue());
+ }
+ for (Map.Entry<String, Integer> entry : stormBase.get_component_executors().entrySet()) {
+ if (!componentExecutors.containsKey(entry.getKey())) {
+ newComponentExecutors.put(entry.getKey(), entry.getValue());
+ }
+ }
+ if (newComponentExecutors.size() > 0)
+ newElems.set_component_executors(newComponentExecutors);
+ }
+
+ Map<String, DebugOptions> ComponentDebug = new HashMap<>();
+ Map<String, DebugOptions> oldComponentDebug = stormBase.get_component_debug();
+
+ Map<String, DebugOptions> newComponentDebug = newElems.get_component_debug();
+ /// oldComponentDebug.keySet()/ newComponentDebug.keySet() maybe be APersistentSet, which don't support addAll
+ Set<String> debugOptionsKeys = new HashSet<>();
+ debugOptionsKeys.addAll(oldComponentDebug.keySet());
+ debugOptionsKeys.addAll(newComponentDebug.keySet());
+ for (String key : debugOptionsKeys) {
+ boolean enable = false;
+ double samplingpct = 0;
+ if (oldComponentDebug.containsKey(key)) {
+ enable = oldComponentDebug.get(key).is_enable();
+ samplingpct = oldComponentDebug.get(key).get_samplingpct();
+ }
+ if (newComponentDebug.containsKey(key)) {
+ enable = newComponentDebug.get(key).is_enable();
+ samplingpct += newComponentDebug.get(key).get_samplingpct();
+ }
+ DebugOptions debugOptions = new DebugOptions();
+ debugOptions.set_enable(enable);
+ debugOptions.set_samplingpct(samplingpct);
+ ComponentDebug.put(key, debugOptions);
+ }
+ if (ComponentDebug.size() > 0) {
+ newElems.set_component_debug(ComponentDebug);
+ }
+
+ if (StringUtils.isBlank(newElems.get_name())) {
+ newElems.set_name(stormBase.get_name());
+ }
+ if (newElems.get_status() == null) {
+ newElems.set_status(stormBase.get_status());
+ }
+ if (newElems.get_num_workers() == 0) {
+ newElems.set_num_workers(stormBase.get_num_workers());
+ }
+ if (newElems.get_launch_time_secs() == 0) {
+ newElems.set_launch_time_secs(stormBase.get_launch_time_secs());
+ }
+ if (StringUtils.isBlank(newElems.get_owner())) {
+ newElems.set_owner(stormBase.get_owner());
+ }
+ if (newElems.get_topology_action_options() == null) {
+ newElems.set_topology_action_options(stormBase.get_topology_action_options());
+ }
+ if (newElems.get_status() == null) {
+ newElems.set_status(stormBase.get_status());
+ }
+ stateStorage.set_data(ClusterUtils.stormPath(stormId), Utils.serialize(newElems), acls);
+ }
+
+ @Override
+ public void removeStormBase(String stormId) {
+ stateStorage.delete_node(ClusterUtils.stormPath(stormId));
+ }
+
+ @Override
+ public void setAssignment(String stormId, Assignment info) {
+ stateStorage.set_data(ClusterUtils.assignmentPath(stormId), Utils.serialize(info), acls);
+ }
+
+ @Override
+ public void setupBlobstore(String key, NimbusInfo nimbusInfo, Integer versionInfo) {
+ String path = ClusterUtils.blobstorePath(key) + ClusterUtils.ZK_SEPERATOR + nimbusInfo.toHostPortString() + "-" + versionInfo;
+ LOG.info("set-path: {}", path);
+ stateStorage.mkdirs(ClusterUtils.blobstorePath(key), acls);
+ stateStorage.delete_node_blobstore(ClusterUtils.blobstorePath(key), nimbusInfo.toHostPortString());
+ stateStorage.set_ephemeral_node(path, null, acls);
+ }
+
+ @Override
+ public List<String> activeKeys() {
+ return stateStorage.get_children(ClusterUtils.BLOBSTORE_SUBTREE, false);
+ }
+
+ // blobstore state
+ @Override
+ public List<String> blobstore(Runnable callback) {
+ if (callback != null) {
+ blobstoreCallback.set(callback);
+ }
+ stateStorage.sync_path(ClusterUtils.BLOBSTORE_SUBTREE);
+ return stateStorage.get_children(ClusterUtils.BLOBSTORE_SUBTREE, callback != null);
+
+ }
+
+ @Override
+ public void removeStorm(String stormId) {
+ stateStorage.delete_node(ClusterUtils.assignmentPath(stormId));
+ stateStorage.delete_node(ClusterUtils.credentialsPath(stormId));
+ stateStorage.delete_node(ClusterUtils.logConfigPath(stormId));
+ stateStorage.delete_node(ClusterUtils.profilerConfigPath(stormId));
+ removeStormBase(stormId);
+ }
+
+ @Override
+ public void removeBlobstoreKey(String blobKey) {
+ LOG.debug("remove key {}", blobKey);
+ stateStorage.delete_node(ClusterUtils.blobstorePath(blobKey));
+ }
+
+ @Override
+ public void removeKeyVersion(String blobKey) {
+ stateStorage.delete_node(ClusterUtils.blobstoreMaxKeySequenceNumberPath(blobKey));
+ }
+
+ @Override
+ public void reportError(String stormId, String componentId, String node, Long port, Throwable error) {
+
+ String path = ClusterUtils.errorPath(stormId, componentId);
+ String lastErrorPath = ClusterUtils.lastErrorPath(stormId, componentId);
+ ErrorInfo errorInfo = new ErrorInfo(ClusterUtils.stringifyError(error), Time.currentTimeSecs());
+ errorInfo.set_host(node);
+ errorInfo.set_port(port.intValue());
+ byte[] serData = Utils.serialize(errorInfo);
+ stateStorage.mkdirs(path, acls);
+ stateStorage.create_sequential(path + ClusterUtils.ZK_SEPERATOR + "e", serData, acls);
+ stateStorage.set_data(lastErrorPath, serData, acls);
+ List<String> childrens = stateStorage.get_children(path, false);
+
+ Collections.sort(childrens, new Comparator<String>() {
+ public int compare(String arg0, String arg1) {
+ return Long.compare(Long.parseLong(arg0.substring(1)), Long.parseLong(arg1.substring(1)));
+ }
+ });
+
+ while (childrens.size() > 10) {
+ stateStorage.delete_node(path + ClusterUtils.ZK_SEPERATOR + childrens.remove(0));
+ }
+ }
+
+ @Override
+ public List<ErrorInfo> errors(String stormId, String componentId) {
+ List<ErrorInfo> errorInfos = new ArrayList<>();
+ String path = ClusterUtils.errorPath(stormId, componentId);
+ if (stateStorage.node_exists(path, false)) {
+ List<String> childrens = stateStorage.get_children(path, false);
+ for (String child : childrens) {
+ String childPath = path + ClusterUtils.ZK_SEPERATOR + child;
+ ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(childPath, false), ErrorInfo.class);
+ if (errorInfo != null)
+ errorInfos.add(errorInfo);
+ }
+ }
+ Collections.sort(errorInfos, new Comparator<ErrorInfo>() {
+ public int compare(ErrorInfo arg0, ErrorInfo arg1) {
+ return Integer.compare(arg1.get_error_time_secs(), arg0.get_error_time_secs());
+ }
+ });
+
+ return errorInfos;
+ }
+
+ @Override
+ public ErrorInfo lastError(String stormId, String componentId) {
+
+ String path = ClusterUtils.lastErrorPath(stormId, componentId);
+ if (stateStorage.node_exists(path, false)) {
+ ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), ErrorInfo.class);
+ return errorInfo;
+ }
+
+ return null;
+ }
+
+ @Override
+ public void setCredentials(String stormId, Credentials creds, Map topoConf) throws NoSuchAlgorithmException {
+ List<ACL> aclList = ClusterUtils.mkTopoOnlyAcls(topoConf);
+ String path = ClusterUtils.credentialsPath(stormId);
+ stateStorage.set_data(path, Utils.serialize(creds), aclList);
+
+ }
+
+ @Override
+ public Credentials credentials(String stormId, Runnable callback) {
+ if (callback != null) {
+ credentialsCallback.put(stormId, callback);
+ }
+ String path = ClusterUtils.credentialsPath(stormId);
+ return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, callback != null), Credentials.class);
+
+ }
+
+ @Override
+ public void disconnect() {
+ stateStorage.unregister(stateId);
+ if (solo)
+ stateStorage.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/VersionedData.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/VersionedData.java b/storm-core/src/jvm/org/apache/storm/cluster/VersionedData.java
new file mode 100644
index 0000000..3de2a88
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/VersionedData.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+public class VersionedData<D> {
+ private final int version;
+ private final D data;
+
+ public VersionedData(int version, D data) {
+ this.version = version;
+ this.data = data;
+ }
+
+ public int getVersion() {
+ return version;
+ }
+
+ public D getData() {
+ return data;
+ }
+}