You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2016/10/27 16:05:22 UTC

[01/21] storm git commit: STORM-2018: Supervisor V2

Repository: storm
Updated Branches:
  refs/heads/1.x-branch e59447bd7 -> 17218ea87


http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
new file mode 100644
index 0000000..9b20ef8
--- /dev/null
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
@@ -0,0 +1,265 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.storm.Config;
+import org.apache.storm.daemon.supervisor.Container.ContainerType;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileRequest;
+import org.junit.Test;
+import org.yaml.snakeyaml.Yaml;
+
+import com.google.common.base.Joiner;
+
+public class ContainerTest {
+    public static class MockContainer extends Container {
+        
+        protected MockContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+                LocalAssignment assignment, String workerId,
+                Map<String, Object> topoConf, AdvancedFSOps ops) throws IOException {
+            super(type, conf, supervisorId, port, assignment, workerId, topoConf, ops);
+        }
+
+        public final List<Long> killedPids = new ArrayList<>();
+        public final List<Long> forceKilledPids = new ArrayList<>();
+        public final Set<Long> allPids = new HashSet<>();
+
+        @Override
+        protected void kill(long pid) {
+            killedPids.add(pid);
+        }
+        
+        @Override
+        protected void forceKill(long pid) {
+            forceKilledPids.add(pid);
+        }
+        
+        @Override
+        protected Set<Long> getAllPids() throws IOException {
+            return allPids;
+        }
+        
+        @Override
+        public void launch() throws IOException {
+            fail("THIS IS NOT UNDER TEST");
+        }
+
+        @Override
+        public void relaunch() throws IOException {
+            fail("THIS IS NOT UNDER TEST");
+        }
+
+        @Override
+        public boolean didMainProcessExit() {
+            fail("THIS IS NOT UNDER TEST");
+            return false;
+        }
+
+        @Override
+        public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException {
+            fail("THIS IS NOT UNDER TEST");
+            return false;
+        }
+    }
+
+    @Test
+    public void testKill() throws Exception {
+        final String topoId = "test_topology";
+        final Map<String, Object> superConf = new HashMap<>();
+        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf, 
+                "SUPERVISOR", 8080, la, "worker", new HashMap<>(), ops);
+        mc.kill();
+        assertEquals(Collections.EMPTY_LIST, mc.killedPids);
+        assertEquals(Collections.EMPTY_LIST, mc.forceKilledPids);
+        mc.forceKill();
+        assertEquals(Collections.EMPTY_LIST, mc.killedPids);
+        assertEquals(Collections.EMPTY_LIST, mc.forceKilledPids);
+        
+        long pid = 987654321;
+        mc.allPids.add(pid);
+        
+        mc.kill();
+        assertEquals(mc.allPids, new HashSet<>(mc.killedPids));
+        assertEquals(Collections.EMPTY_LIST, mc.forceKilledPids);
+        mc.killedPids.clear();
+        
+        mc.forceKill();
+        assertEquals(Collections.EMPTY_LIST, mc.killedPids);
+        assertEquals(mc.allPids, new HashSet<>(mc.forceKilledPids));
+    }
+    
+    private static final Joiner PATH_JOIN = Joiner.on(File.separator).skipNulls();
+    private static final String DOUBLE_SEP = File.separator + File.separator;    
+    static String asAbsPath(String ... parts) {
+        return (File.separator + PATH_JOIN.join(parts)).replace(DOUBLE_SEP, File.separator);
+    }
+    
+    static File asAbsFile(String ... parts) {
+        return new File(asAbsPath(parts));
+    }
+    
+    static String asPath(String ... parts) {
+        return PATH_JOIN.join(parts);
+    }
+    
+    public static File asFile(String ... parts) {
+        return new File(asPath(parts));
+    }
+    
+    @SuppressWarnings("unchecked")
+    @Test
+    public void testSetup() throws Exception {
+        final int port = 8080;
+        final String topoId = "test_topology";
+        final String workerId = "worker_id";
+        final String user = "me";
+        final String stormLocal = asAbsPath("tmp", "testing");
+        final File workerArtifacts = asAbsFile(stormLocal, topoId, String.valueOf(port));
+        final File logMetadataFile = new File(workerArtifacts, "worker.yaml");
+        final File workerUserFile = asAbsFile(stormLocal, "workers-users", workerId);
+        final File workerRoot = asAbsFile(stormLocal, "workers", workerId);
+        final File distRoot = asAbsFile(stormLocal, "supervisor", "stormdist", topoId);
+        
+        final Map<String, Object> topoConf = new HashMap<>();
+        final List<String> topoUsers = Arrays.asList("t-user-a", "t-user-b");
+        final List<String> logUsers = Arrays.asList("l-user-a", "l-user-b");
+        
+        final List<String> topoGroups = Arrays.asList("t-group-a", "t-group-b");
+        final List<String> logGroups = Arrays.asList("l-group-a", "l-group-b");
+        
+        topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, user);
+        topoConf.put(Config.LOGS_GROUPS, logGroups);
+        topoConf.put(Config.TOPOLOGY_GROUPS, topoGroups);
+        topoConf.put(Config.LOGS_USERS, logUsers);
+        topoConf.put(Config.TOPOLOGY_USERS, topoUsers);
+        
+        final Map<String, Object> superConf = new HashMap<>();
+        superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
+        superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
+        
+        final StringWriter yamlDump = new StringWriter();
+        
+        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+        when(ops.fileExists(workerArtifacts)).thenReturn(true);
+        when(ops.fileExists(workerRoot)).thenReturn(true);
+        when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump);
+        
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf, 
+                "SUPERVISOR", 8080, la, workerId, topoConf, ops);
+        
+        mc.setup();
+        
+        //Initial Setup
+        verify(ops).forceMkdir(new File(workerRoot, "pids"));
+        verify(ops).forceMkdir(new File(workerRoot, "tmp"));
+        verify(ops).forceMkdir(new File(workerRoot, "heartbeats"));
+        verify(ops).fileExists(workerArtifacts);
+        
+        //Log file permissions
+        verify(ops).getWriter(logMetadataFile);
+        
+        String yamlResult = yamlDump.toString();
+        Yaml yaml = new Yaml();
+        Map<String, Object> result = (Map<String, Object>) yaml.load(yamlResult);
+        assertEquals(workerId, result.get("worker-id"));
+        assertEquals(user, result.get(Config.TOPOLOGY_SUBMITTER_USER));
+        HashSet<String> allowedUsers = new HashSet<>(topoUsers);
+        allowedUsers.addAll(logUsers);
+        assertEquals(allowedUsers, new HashSet<String>((List<String>)result.get(Config.LOGS_USERS)));
+        
+        HashSet<String> allowedGroups = new HashSet<>(topoGroups);
+        allowedGroups.addAll(logGroups);
+        assertEquals(allowedGroups, new HashSet<String>((List<String>)result.get(Config.LOGS_GROUPS)));
+        
+        //Save the current user to help with recovery
+        verify(ops).dump(workerUserFile, user);
+        
+        //Create links to artifacts dir
+        verify(ops).createSymlink(new File(workerRoot, "artifacts"), workerArtifacts);
+        
+        //Create links to blobs 
+        verify(ops).createSymlink(new File(workerRoot, "resources"), new File(distRoot, "resources"));
+    }
+    
+    @Test
+    public void testCleanup() throws Exception {
+        final int port = 8080;
+        final long pid = 100;
+        final String topoId = "test_topology";
+        final String workerId = "worker_id";
+        final String user = "me";
+        final String stormLocal = asAbsPath("tmp", "testing");
+        final File workerArtifacts = asAbsFile(stormLocal, topoId, String.valueOf(port));
+        final File logMetadataFile = new File(workerArtifacts, "worker.yaml");
+        final File workerUserFile = asAbsFile(stormLocal, "workers-users", workerId);
+        final File workerRoot = asAbsFile(stormLocal, "workers", workerId);
+        final File workerPidsRoot = new File(workerRoot, "pids");
+        
+        final Map<String, Object> topoConf = new HashMap<>();
+        topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, user);
+        
+        final Map<String, Object> superConf = new HashMap<>();
+        superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
+        superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
+        
+        final StringWriter yamlDump = new StringWriter();
+        
+        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+        when(ops.fileExists(workerArtifacts)).thenReturn(true);
+        when(ops.fileExists(workerRoot)).thenReturn(true);
+        when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump);
+        
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf, 
+                "SUPERVISOR", port, la, workerId, topoConf, ops);
+        mc.allPids.add(pid);
+        
+        mc.cleanUp();
+        verify(ops).deleteIfExists(eq(new File(workerPidsRoot, String.valueOf(pid))), eq(user), any(String.class));
+        
+        verify(ops).deleteIfExists(eq(new File(workerRoot, "pids")), eq(user), any(String.class));
+        verify(ops).deleteIfExists(eq(new File(workerRoot, "tmp")), eq(user), any(String.class));
+        verify(ops).deleteIfExists(eq(new File(workerRoot, "heartbeats")), eq(user), any(String.class));
+        verify(ops).deleteIfExists(eq(workerRoot), eq(user), any(String.class));
+        verify(ops).deleteIfExists(workerUserFile);
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java
new file mode 100644
index 0000000..24ccda5
--- /dev/null
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/SlotTest.java
@@ -0,0 +1,515 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.storm.daemon.supervisor.Slot.StaticState;
+import org.apache.storm.daemon.supervisor.Slot.TopoProfileAction;
+import org.apache.storm.daemon.supervisor.Slot.DynamicState;
+import org.apache.storm.daemon.supervisor.Slot.MachineState;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.NodeInfo;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.generated.WorkerResources;
+import org.apache.storm.localizer.ILocalizer;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Time;
+import org.junit.Test;
+
+public class SlotTest {
+    static WorkerResources mkWorkerResources(Double cpu, Double mem_on_heap, Double mem_off_heap) {
+        WorkerResources resources = new WorkerResources();
+        if (cpu != null) {
+            resources.set_cpu(cpu);
+        }
+        
+        if (mem_on_heap != null) {
+            resources.set_mem_on_heap(mem_on_heap);
+        }
+        
+        if (mem_off_heap != null) {
+            resources.set_mem_off_heap(mem_off_heap);
+        }
+        return resources;
+    }
+    
+    static LSWorkerHeartbeat mkWorkerHB(String id, int port, List<ExecutorInfo> exec, Integer timeSecs) {
+        LSWorkerHeartbeat ret = new LSWorkerHeartbeat();
+        ret.set_topology_id(id);
+        ret.set_port(port);
+        ret.set_executors(exec);
+        ret.set_time_secs(timeSecs);
+        return ret;
+    }
+    
+    static List<ExecutorInfo> mkExecutorInfoList(int ... executors) {
+        ArrayList<ExecutorInfo> ret = new ArrayList<>(executors.length);
+        for (int exec : executors) {
+            ExecutorInfo execInfo = new ExecutorInfo();
+            execInfo.set_task_start(exec);
+            execInfo.set_task_end(exec);
+            ret.add(execInfo);
+        }
+        return ret;
+    }
+    
+    static LocalAssignment mkLocalAssignment(String id, List<ExecutorInfo> exec, WorkerResources resources) {
+        LocalAssignment ret = new LocalAssignment();
+        ret.set_topology_id(id);
+        ret.set_executors(exec);
+        if (resources != null) {
+            ret.set_resources(resources);
+        }
+        return ret;
+    }
+    
+    @Test
+    public void testEquivilant() {
+        LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 100.0, 100.0));
+        LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 200.0, 100.0));
+        LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1,2,3,4,5,6), mkWorkerResources(100.0, 100.0, 100.0));
+        LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6,5,4,3,2,1), mkWorkerResources(100.0, 100.0, 100.0));
+
+        assertTrue(Slot.equivalent(null, null));
+        assertTrue(Slot.equivalent(a, a));
+        assertTrue(Slot.equivalent(b, bReordered));
+        assertTrue(Slot.equivalent(bReordered, b));
+        
+        assertFalse(Slot.equivalent(a, aResized));
+        assertFalse(Slot.equivalent(aResized, a));
+        assertFalse(Slot.equivalent(a, null));
+        assertFalse(Slot.equivalent(null, b));
+        assertFalse(Slot.equivalent(a, b));
+    }
+    
+    @Test
+    public void testEmptyToEmpty() throws Exception {
+        Time.startSimulatingAutoAdvanceOnSleep(1010);
+        try {
+            ILocalizer localizer = mock(ILocalizer.class);
+            LocalState state = mock(LocalState.class);
+            ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+            ISupervisor iSuper = mock(ISupervisor.class);
+            StaticState staticState = new StaticState(localizer, 1000, 1000, 1000, 1000,
+                    containerLauncher, "localhost", 8080, iSuper, state);
+            DynamicState dynamicState = new DynamicState(null, null, null);
+            DynamicState nextState = Slot.handleEmpty(dynamicState, staticState);
+            assertEquals(MachineState.EMPTY, nextState.state);
+            assertTrue(Time.currentTimeMillis() > 1000);
+        } finally {
+            Time.stopSimulating();
+        }
+    }
+    
+    @Test
+    public void testLaunchContainerFromEmpty() throws Exception {
+        Time.startSimulatingAutoAdvanceOnSleep(1010);
+        try {
+            int port = 8080;
+            String topoId = "NEW";
+            List<ExecutorInfo> execList =  mkExecutorInfoList(1,2,3,4,5);
+            LocalAssignment newAssignment = 
+                    mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
+            
+            ILocalizer localizer = mock(ILocalizer.class);
+            Container container = mock(Container.class);
+            LocalState state = mock(LocalState.class);
+            ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+            when(containerLauncher.launchContainer(port, newAssignment, state)).thenReturn(container);
+            LSWorkerHeartbeat hb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs());
+            when(container.readHeartbeat()).thenReturn(hb, hb);
+            
+            @SuppressWarnings("unchecked")
+            Future<Void> baseFuture = mock(Future.class);
+            when(localizer.requestDownloadBaseTopologyBlobs(newAssignment, port)).thenReturn(baseFuture);
+            
+            @SuppressWarnings("unchecked")
+            Future<Void> blobFuture = mock(Future.class);
+            when(localizer.requestDownloadTopologyBlobs(newAssignment, port)).thenReturn(blobFuture);
+            
+            ISupervisor iSuper = mock(ISupervisor.class);
+            StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+                    containerLauncher, "localhost", port, iSuper, state);
+            DynamicState dynamicState = new DynamicState(null, null, null)
+                    .withNewAssignment(newAssignment);
+            
+            DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+            verify(localizer).requestDownloadBaseTopologyBlobs(newAssignment, port);
+            assertEquals(MachineState.WAITING_FOR_BASIC_LOCALIZATION, nextState.state);
+            assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
+            assertEquals(newAssignment, nextState.pendingLocalization);
+            assertEquals(0, Time.currentTimeMillis());
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            verify(baseFuture).get(1000, TimeUnit.MILLISECONDS);
+            verify(localizer).requestDownloadTopologyBlobs(newAssignment, port);
+            assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
+            assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
+            assertEquals(newAssignment, nextState.pendingLocalization);
+            assertEquals(0, Time.currentTimeMillis());
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
+            verify(containerLauncher).launchContainer(port, newAssignment, state);
+            assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
+            assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+            assertSame(null, nextState.pendingLocalization);
+            assertSame(newAssignment, nextState.currentAssignment);
+            assertSame(container, nextState.container);
+            assertEquals(0, Time.currentTimeMillis());
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+            assertSame(null, nextState.pendingLocalization);
+            assertSame(newAssignment, nextState.currentAssignment);
+            assertSame(container, nextState.container);
+            assertEquals(0, Time.currentTimeMillis());
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+            assertSame(null, nextState.pendingLocalization);
+            assertSame(newAssignment, nextState.currentAssignment);
+            assertSame(container, nextState.container);
+            assertTrue(Time.currentTimeMillis() > 1000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+            assertSame(null, nextState.pendingLocalization);
+            assertSame(newAssignment, nextState.currentAssignment);
+            assertSame(container, nextState.container);
+            assertTrue(Time.currentTimeMillis() > 2000);
+        } finally {
+            Time.stopSimulating();
+        }
+    }
+
+
+    @Test
+    public void testRelaunch() throws Exception {
+        Time.startSimulatingAutoAdvanceOnSleep(1010);
+        try {
+            int port = 8080;
+            String topoId = "CURRENT";
+            List<ExecutorInfo> execList =  mkExecutorInfoList(1,2,3,4,5);
+            LocalAssignment assignment = 
+                    mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
+            
+            ILocalizer localizer = mock(ILocalizer.class);
+            Container container = mock(Container.class);
+            ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+            LSWorkerHeartbeat oldhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs()-10);
+            LSWorkerHeartbeat goodhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs());
+            when(container.readHeartbeat()).thenReturn(oldhb, oldhb, goodhb, goodhb);
+            when(container.areAllProcessesDead()).thenReturn(false, true);
+            
+            ISupervisor iSuper = mock(ISupervisor.class);
+            LocalState state = mock(LocalState.class);
+            StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+                    containerLauncher, "localhost", port, iSuper, state);
+            DynamicState dynamicState = new DynamicState(assignment, container, assignment);
+            
+            DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+            assertEquals(MachineState.KILL_AND_RELAUNCH, nextState.state);
+            verify(container).kill();
+            assertTrue(Time.currentTimeMillis() > 1000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.KILL_AND_RELAUNCH, nextState.state);
+            verify(container).forceKill();
+            assertTrue(Time.currentTimeMillis() > 2000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
+            verify(container).relaunch();
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
+            assertTrue(Time.currentTimeMillis() > 3000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+        } finally {
+            Time.stopSimulating();
+        }
+    }
+    
+    @Test
+    public void testReschedule() throws Exception {
+        Time.startSimulatingAutoAdvanceOnSleep(1010);
+        try {
+            int port = 8080;
+            String cTopoId = "CURRENT";
+            List<ExecutorInfo> cExecList =  mkExecutorInfoList(1,2,3,4,5);
+            LocalAssignment cAssignment = 
+                    mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
+            
+            Container cContainer = mock(Container.class);
+            LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
+            when(cContainer.readHeartbeat()).thenReturn(chb);
+            when(cContainer.areAllProcessesDead()).thenReturn(false, true);
+            
+            String nTopoId = "NEW";
+            List<ExecutorInfo> nExecList =  mkExecutorInfoList(1,2,3,4,5);
+            LocalAssignment nAssignment = 
+                    mkLocalAssignment(nTopoId, nExecList, mkWorkerResources(100.0, 100.0, 100.0));
+            
+            ILocalizer localizer = mock(ILocalizer.class);
+            Container nContainer = mock(Container.class);
+            LocalState state = mock(LocalState.class);
+            ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+            when(containerLauncher.launchContainer(port, nAssignment, state)).thenReturn(nContainer);
+            LSWorkerHeartbeat nhb = mkWorkerHB(nTopoId, 100, nExecList, Time.currentTimeSecs());
+            when(nContainer.readHeartbeat()).thenReturn(nhb, nhb);
+            
+            @SuppressWarnings("unchecked")
+            Future<Void> baseFuture = mock(Future.class);
+            when(localizer.requestDownloadBaseTopologyBlobs(nAssignment, port)).thenReturn(baseFuture);
+            
+            @SuppressWarnings("unchecked")
+            Future<Void> blobFuture = mock(Future.class);
+            when(localizer.requestDownloadTopologyBlobs(nAssignment, port)).thenReturn(blobFuture);
+            
+            ISupervisor iSuper = mock(ISupervisor.class);
+            StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+                    containerLauncher, "localhost", port, iSuper, state);
+            DynamicState dynamicState = new DynamicState(cAssignment, cContainer, nAssignment);
+            
+            DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+            assertEquals(MachineState.KILL, nextState.state);
+            verify(cContainer).kill();
+            verify(localizer).requestDownloadBaseTopologyBlobs(nAssignment, port);
+            assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
+            assertEquals(nAssignment, nextState.pendingLocalization);
+            assertTrue(Time.currentTimeMillis() > 1000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.KILL, nextState.state);
+            verify(cContainer).forceKill();
+            assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
+            assertEquals(nAssignment, nextState.pendingLocalization);
+            assertTrue(Time.currentTimeMillis() > 2000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.WAITING_FOR_BASIC_LOCALIZATION, nextState.state);
+            verify(cContainer).cleanUp();
+            verify(localizer).releaseSlotFor(cAssignment, port);
+            assertTrue(Time.currentTimeMillis() > 2000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
+            verify(baseFuture).get(1000, TimeUnit.MILLISECONDS);
+            verify(localizer).requestDownloadTopologyBlobs(nAssignment, port);
+            assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
+            assertEquals(nAssignment, nextState.pendingLocalization);
+            assertTrue(Time.currentTimeMillis() > 2000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
+            verify(containerLauncher).launchContainer(port, nAssignment, state);
+            assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
+            assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+            assertSame(null, nextState.pendingLocalization);
+            assertSame(nAssignment, nextState.currentAssignment);
+            assertSame(nContainer, nextState.container);
+            assertTrue(Time.currentTimeMillis() > 2000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+            assertSame(null, nextState.pendingLocalization);
+            assertSame(nAssignment, nextState.currentAssignment);
+            assertSame(nContainer, nextState.container);
+            assertTrue(Time.currentTimeMillis() > 2000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+            assertSame(null, nextState.pendingLocalization);
+            assertSame(nAssignment, nextState.currentAssignment);
+            assertSame(nContainer, nextState.container);
+            assertTrue(Time.currentTimeMillis() > 3000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertSame("pendingDownload is not null", null, nextState.pendingDownload);
+            assertSame(null, nextState.pendingLocalization);
+            assertSame(nAssignment, nextState.currentAssignment);
+            assertSame(nContainer, nextState.container);
+            assertTrue(Time.currentTimeMillis() > 4000);
+        } finally {
+            Time.stopSimulating();
+        }
+    }
+
+    
+    @Test
+    public void testRunningToEmpty() throws Exception {
+        Time.startSimulatingAutoAdvanceOnSleep(1010);
+        try {
+            int port = 8080;
+            String cTopoId = "CURRENT";
+            List<ExecutorInfo> cExecList =  mkExecutorInfoList(1,2,3,4,5);
+            LocalAssignment cAssignment = 
+                    mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
+            
+            Container cContainer = mock(Container.class);
+            LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
+            when(cContainer.readHeartbeat()).thenReturn(chb);
+            when(cContainer.areAllProcessesDead()).thenReturn(false, true);
+            
+            ILocalizer localizer = mock(ILocalizer.class);
+            ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+            
+            ISupervisor iSuper = mock(ISupervisor.class);
+            LocalState state = mock(LocalState.class);
+            StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+                    containerLauncher, "localhost", port, iSuper, state);
+            DynamicState dynamicState = new DynamicState(cAssignment, cContainer, null);
+            
+            DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+            assertEquals(MachineState.KILL, nextState.state);
+            verify(cContainer).kill();
+            verify(localizer, never()).requestDownloadBaseTopologyBlobs(null, port);
+            assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
+            assertEquals(null, nextState.pendingLocalization);
+            assertTrue(Time.currentTimeMillis() > 1000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.KILL, nextState.state);
+            verify(cContainer).forceKill();
+            assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
+            assertEquals(null, nextState.pendingLocalization);
+            assertTrue(Time.currentTimeMillis() > 2000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.EMPTY, nextState.state);
+            verify(cContainer).cleanUp();
+            verify(localizer).releaseSlotFor(cAssignment, port);
+            assertEquals(null, nextState.container);
+            assertEquals(null, nextState.currentAssignment);
+            assertTrue(Time.currentTimeMillis() > 2000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.EMPTY, nextState.state);
+            assertEquals(null, nextState.container);
+            assertEquals(null, nextState.currentAssignment);
+            assertTrue(Time.currentTimeMillis() > 3000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.EMPTY, nextState.state);
+            assertEquals(null, nextState.container);
+            assertEquals(null, nextState.currentAssignment);
+            assertTrue(Time.currentTimeMillis() > 3000);
+        } finally {
+            Time.stopSimulating();
+        }
+    }
+    
+    @Test
+    public void testRunWithProfileActions() throws Exception {
+        Time.startSimulatingAutoAdvanceOnSleep(1010);
+        try {
+            int port = 8080;
+            String cTopoId = "CURRENT";
+            List<ExecutorInfo> cExecList =  mkExecutorInfoList(1,2,3,4,5);
+            LocalAssignment cAssignment = 
+                    mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
+            
+            Container cContainer = mock(Container.class);
+            LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs()+100); //NOT going to timeout for a while
+            when(cContainer.readHeartbeat()).thenReturn(chb, chb, chb, chb, chb, chb);
+            when(cContainer.runProfiling(any(ProfileRequest.class), anyBoolean())).thenReturn(true);
+            
+            ILocalizer localizer = mock(ILocalizer.class);
+            ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+            
+            ISupervisor iSuper = mock(ISupervisor.class);
+            LocalState state = mock(LocalState.class);
+            StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+                    containerLauncher, "localhost", port, iSuper, state);
+            Set<TopoProfileAction> profileActions = new HashSet<>();
+            ProfileRequest request = new ProfileRequest();
+            request.set_action(ProfileAction.JPROFILE_STOP);
+            NodeInfo info = new NodeInfo();
+            info.set_node("localhost");
+            info.add_to_port(port);
+            request.set_nodeInfo(info);
+            request.set_time_stamp(Time.currentTimeMillis() + 3000);//3 seconds from now
+            
+            TopoProfileAction profile = new TopoProfileAction(cTopoId, request);
+            profileActions.add(profile);
+            Set<TopoProfileAction> expectedPending = new HashSet<>();
+            expectedPending.add(profile);
+            
+            
+            DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment).withProfileActions(profileActions, Collections.<TopoProfileAction> emptySet());
+            
+            DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            verify(cContainer).runProfiling(request, false);
+            assertEquals(expectedPending, nextState.pendingStopProfileActions);
+            assertEquals(expectedPending, nextState.profileActions);
+            assertTrue(Time.currentTimeMillis() > 1000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertEquals(expectedPending, nextState.pendingStopProfileActions);
+            assertEquals(expectedPending, nextState.profileActions);
+            assertTrue(Time.currentTimeMillis() > 2000);
+            
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertEquals(expectedPending, nextState.pendingStopProfileActions);
+            assertEquals(expectedPending, nextState.profileActions);
+            assertTrue(Time.currentTimeMillis() > 3000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            verify(cContainer).runProfiling(request, true);
+            assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.pendingStopProfileActions);
+            assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.profileActions);
+            assertTrue(Time.currentTimeMillis() > 4000);
+            
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.pendingStopProfileActions);
+            assertEquals(Collections.<TopoProfileAction> emptySet(), nextState.profileActions);
+            assertTrue(Time.currentTimeMillis() > 5000);
+        } finally {
+            Time.stopSimulating();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java b/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
new file mode 100644
index 0000000..41c17c8
--- /dev/null
+++ b/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.localizer;
+
+import static org.mockito.Matchers.*;
+import static org.mockito.Mockito.*;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.storm.daemon.supervisor.AdvancedFSOps;
+import org.apache.storm.utils.ConfigUtils;
+import org.junit.Test;
+
+import org.apache.storm.Config;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.localizer.LocalizedResource;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.security.auth.DefaultPrincipalToLocal;
+import org.apache.storm.utils.Utils;
+
+public class AsyncLocalizerTest {
+
+    @Test
+    public void testRequestDownloadBaseTopologyBlobs() throws Exception {
+        final String topoId = "TOPO";
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        ExecutorInfo ei = new ExecutorInfo();
+        ei.set_task_start(1);
+        ei.set_task_end(1);
+        la.add_to_executors(ei);
+        final int port = 8080;
+        final String jarKey = topoId + "-stormjar.jar";
+        final String codeKey = topoId + "-stormcode.ser";
+        final String confKey = topoId + "-stormconf.ser";
+        final String stormLocal = "/tmp/storm-local/";
+        final String stormRoot = stormLocal+topoId+"/";
+        final File fStormRoot = new File(stormRoot);
+        ClientBlobStore blobStore = mock(ClientBlobStore.class);
+        Map<String, Object> conf = new HashMap<>();
+        conf.put(Config.SUPERVISOR_BLOBSTORE, ClientBlobStore.class.getName());
+        conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName());
+        conf.put(Config.STORM_CLUSTER_MODE, "distributed");
+        conf.put(Config.STORM_LOCAL_DIR, stormLocal);
+        Localizer localizer = mock(Localizer.class);
+        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        ConfigUtils mockedCU = mock(ConfigUtils.class);
+        Utils mockedU = mock(Utils.class);
+        
+        Map<String, Object> topoConf = new HashMap<>(conf);
+        
+        AsyncLocalizer al = new AsyncLocalizer(conf, localizer, ops);
+        ConfigUtils orig = ConfigUtils.setInstance(mockedCU);
+        Utils origUtils = Utils.setInstance(mockedU);
+        try {
+            when(mockedCU.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot);
+            when(mockedCU.supervisorLocalDirImpl(conf)).thenReturn(stormLocal);
+            when(mockedU.newInstanceImpl(ClientBlobStore.class)).thenReturn(blobStore);
+            when(mockedCU.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf);
+
+            Future<Void> f = al.requestDownloadBaseTopologyBlobs(la, port);
+            f.get(20, TimeUnit.SECONDS);
+            // We should be done now...
+            
+            verify(blobStore).prepare(conf);
+            verify(mockedU).downloadResourcesAsSupervisorImpl(eq(jarKey), startsWith(stormLocal), eq(blobStore));
+            verify(mockedU).downloadResourcesAsSupervisorImpl(eq(codeKey), startsWith(stormLocal), eq(blobStore));
+            verify(mockedU).downloadResourcesAsSupervisorImpl(eq(confKey), startsWith(stormLocal), eq(blobStore));
+            verify(blobStore).shutdown();
+            //Extracting the dir from the jar
+            verify(mockedU).extractDirFromJarImpl(endsWith("stormjar.jar"), eq("resources"), any(File.class));
+            verify(ops).moveDirectoryPreferAtomic(any(File.class), eq(fStormRoot));
+            verify(ops).setupStormCodeDir(topoConf, fStormRoot);
+            
+            verify(ops, never()).deleteIfExists(any(File.class));
+        } finally {
+            al.shutdown();
+            ConfigUtils.setInstance(orig);
+            Utils.setInstance(origUtils);
+        }
+    }
+
+    @Test
+    public void testRequestDownloadTopologyBlobs() throws Exception {
+        final String topoId = "TOPO-12345";
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        ExecutorInfo ei = new ExecutorInfo();
+        ei.set_task_start(1);
+        ei.set_task_end(1);
+        la.add_to_executors(ei);
+        final String topoName = "TOPO";
+        final int port = 8080;
+        final String user = "user";
+        final String simpleLocalName = "simple.txt";
+        final String simpleKey = "simple";
+        
+        final String stormLocal = "/tmp/storm-local/";
+        final File userDir = new File(stormLocal, user);
+        final String stormRoot = stormLocal+topoId+"/";
+        
+        final String localizerRoot = "/tmp/storm-localizer/";
+        final String simpleLocalFile = localizerRoot + user + "/simple";
+        final String simpleCurrentLocalFile = localizerRoot + user + "/simple.current";
+       
+        final StormTopology st = new StormTopology();
+        st.set_spouts(new HashMap<>());
+        st.set_bolts(new HashMap<>());
+        st.set_state_spouts(new HashMap<>());
+ 
+        Map<String, Map<String, Object>> topoBlobMap = new HashMap<>();
+        Map<String, Object> simple = new HashMap<>();
+        simple.put("localname", simpleLocalName);
+        simple.put("uncompress", false);
+        topoBlobMap.put(simpleKey, simple);
+        
+        Map<String, Object> conf = new HashMap<>();
+        conf.put(Config.STORM_LOCAL_DIR, stormLocal);
+        Localizer localizer = mock(Localizer.class);
+        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        ConfigUtils mockedCU = mock(ConfigUtils.class);
+        Utils mockedU = mock(Utils.class);
+        
+        Map<String, Object> topoConf = new HashMap<>(conf);
+        topoConf.put(Config.TOPOLOGY_BLOBSTORE_MAP, topoBlobMap);
+        topoConf.put(Config.TOPOLOGY_SUBMITTER_USER, user);
+        topoConf.put(Config.TOPOLOGY_NAME, topoName);
+        
+        List<LocalizedResource> localizedList = new ArrayList<>();
+        LocalizedResource simpleLocal = new LocalizedResource(simpleKey, simpleLocalFile, false);
+        localizedList.add(simpleLocal);
+        
+        AsyncLocalizer al = new AsyncLocalizer(conf, localizer, ops);
+        ConfigUtils orig = ConfigUtils.setInstance(mockedCU);
+        Utils origUtils = Utils.setInstance(mockedU);
+        try {
+            when(mockedCU.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot);
+            when(mockedCU.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf);
+            when(mockedCU.readSupervisorTopologyImpl(conf, topoId, ops)).thenReturn(st);
+            
+            when(localizer.getLocalUserFileCacheDir(user)).thenReturn(userDir);
+            
+            when(localizer.getBlobs(any(List.class), eq(user), eq(topoName), eq(userDir))).thenReturn(localizedList);
+            
+            Future<Void> f = al.requestDownloadTopologyBlobs(la, port);
+            f.get(20, TimeUnit.SECONDS);
+            // We should be done now...
+            
+            verify(localizer).getLocalUserFileCacheDir(user);
+            verify(ops).fileExists(userDir);
+            verify(ops).forceMkdir(userDir);
+            
+            verify(localizer).getBlobs(any(List.class), eq(user), eq(topoName), eq(userDir));
+
+            verify(ops).createSymlink(new File(stormRoot, simpleLocalName), new File(simpleCurrentLocalFile));
+        } finally {
+            al.shutdown();
+            ConfigUtils.setInstance(orig);
+            Utils.setInstance(origUtils);
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java b/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
index ab2c9af..04b5ab2 100644
--- a/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
+++ b/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSetTest.java
@@ -34,8 +34,8 @@ public class LocalizedResourceRetentionSetTest {
     // check adding reference to local resource with topology of same name
     localresource2.addReference(("topo2"));
 
-    lrset.addResource("key1", localresource1, false);
-    lrset.addResource("key2", localresource2, false);
+    lrset.add("key1", localresource1, false);
+    lrset.add("key2", localresource2, false);
     lrretset.addResources(lrset);
     assertEquals("number to clean is not 0", 0, lrretset.getSizeWithNoReferences());
     localresource1.removeReference(("topo1"));
@@ -64,9 +64,9 @@ public class LocalizedResourceRetentionSetTest {
     // check adding reference to local resource with topology of same name
     localresource2.addReference(("topo1"));
     localresource2.setSize(10);
-    lrset.addResource("key1", localresource1, false);
-    lrset.addResource("key2", localresource2, false);
-    lrset.addResource("archive1", archiveresource1, true);
+    lrset.add("key1", localresource1, false);
+    lrset.add("key2", localresource2, false);
+    lrset.add("archive1", archiveresource1, true);
 
     lrretset.addResources(lrset);
     assertEquals("number to clean is not 2", 2, lrretset.getSizeWithNoReferences());

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceSetTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceSetTest.java b/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceSetTest.java
index 7f20d19..550d695 100644
--- a/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceSetTest.java
+++ b/storm-core/test/jvm/org/apache/storm/localizer/LocalizedResourceSetTest.java
@@ -37,9 +37,9 @@ public class LocalizedResourceSetTest {
     LocalizedResource localresource1 = new LocalizedResource("key1", "testfile1", false, "topo1");
     LocalizedResource localresource2 = new LocalizedResource("key2", "testfile2", true, "topo1");
     assertEquals("size is wrong", 0, lrset.getSize());
-    lrset.addResource("key1", localresource1, false);
+    lrset.add("key1", localresource1, false);
     assertEquals("size is wrong", 1, lrset.getSize());
-    lrset.addResource("key2", localresource2, true);
+    lrset.add("key2", localresource2, true);
     assertEquals("size is wrong", 2, lrset.getSize());
   }
 
@@ -48,8 +48,8 @@ public class LocalizedResourceSetTest {
     LocalizedResourceSet lrset = new LocalizedResourceSet(user1);
     LocalizedResource localresource1 = new LocalizedResource("key1", "testfile1", false, "topo1");
     LocalizedResource localresource2 = new LocalizedResource("key2", "testfile2", true, "topo1");
-    lrset.addResource("key1", localresource1, false);
-    lrset.addResource("key2", localresource2, true);
+    lrset.add("key1", localresource1, false);
+    lrset.add("key2", localresource2, true);
     assertEquals("get doesn't return same object", localresource1, lrset.get("key1", false));
     assertEquals("get doesn't return same object", localresource2, lrset.get("key2", true));
 
@@ -60,8 +60,8 @@ public class LocalizedResourceSetTest {
     LocalizedResourceSet lrset = new LocalizedResourceSet(user1);
     LocalizedResource localresource1 = new LocalizedResource("key1", "testfile1", false, "topo1");
     LocalizedResource localresource2 = new LocalizedResource("key2", "testfile2", true, "topo1");
-    lrset.addResource("key1", localresource1, false);
-    lrset.addResource("key2", localresource2, true);
+    lrset.add("key1", localresource1, false);
+    lrset.add("key2", localresource2, true);
     assertEquals("doesn't exist", true, lrset.exists("key1", false));
     assertEquals("doesn't exist", true, lrset.exists("key2", true));
     boolean val = lrset.remove(localresource1);

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/localizer/LocalizerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/localizer/LocalizerTest.java b/storm-core/test/jvm/org/apache/storm/localizer/LocalizerTest.java
index 45ba108..613e165 100644
--- a/storm-core/test/jvm/org/apache/storm/localizer/LocalizerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/localizer/LocalizerTest.java
@@ -259,7 +259,7 @@ public class LocalizerTest {
 
   // archive passed in must contain symlink named tmptestsymlink if not a zip file
   public void testArchives(String archivePath, boolean supportSymlinks, int size) throws Exception {
-    if (isOnWindows()) {
+    if (Utils.isOnWindows()) {
       // Windows should set this to false cause symlink in compressed file doesn't work properly.
       supportSymlinks = false;
     }
@@ -669,11 +669,4 @@ public class LocalizerTest {
     assertEquals("blob version not correct", 3, Utils.localVersionOfBlob(keyFile.toString()));
     assertTrue("blob file with version 3 not created", new File(keyFile + ".3").exists());
   }
-
-  private boolean isOnWindows() {
-    if (System.getenv("OS") != null) {
-      return System.getenv("OS").equals("Windows_NT");
-    }
-    return false;
-  }
 }


[09/21] storm git commit: STORM-2018: Supervisor V2

Posted by ka...@apache.org.
STORM-2018: Supervisor V2


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/318ab5f3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/318ab5f3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/318ab5f3

Branch: refs/heads/1.x-branch
Commit: 318ab5f300e0820ab862cbeca04b7cb67699b938
Parents: 58cbfe5
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Sep 20 14:59:07 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Sep 20 16:54:14 2016 -0500

----------------------------------------------------------------------
 bin/storm.py                                    |    6 +-
 log4j2/cluster.xml                              |    2 +-
 pom.xml                                         |    4 +-
 .../cluster_state/zookeeper_state_factory.clj   |    5 +-
 .../org/apache/storm/command/kill_workers.clj   |   33 -
 .../clj/org/apache/storm/daemon/executor.clj    |    4 +-
 .../apache/storm/daemon/local_supervisor.clj    |   28 +
 .../clj/org/apache/storm/daemon/logviewer.clj   |    7 +-
 .../clj/org/apache/storm/daemon/supervisor.clj  | 1272 ------------------
 .../src/clj/org/apache/storm/daemon/worker.clj  |    1 +
 storm-core/src/clj/org/apache/storm/event.clj   |   71 -
 .../storm/pacemaker/pacemaker_state_factory.clj |    6 +-
 storm-core/src/clj/org/apache/storm/testing.clj |   74 +-
 .../jvm/org/apache/storm/ProcessSimulator.java  |   82 ++
 .../src/jvm/org/apache/storm/StormTimer.java    |  242 ++++
 .../storm/callback/DefaultWatcherCallBack.java  |   35 +
 .../apache/storm/callback/WatcherCallBack.java  |   24 +
 .../storm/callback/ZKStateChangedCallback.java  |   25 +
 .../storm/cluster/ClusterStateFactory.java      |    4 +-
 .../org/apache/storm/cluster/ClusterUtils.java  |  244 ++++
 .../org/apache/storm/cluster/ExecutorBeat.java  |   44 +
 .../org/apache/storm/cluster/IStateStorage.java |  219 +++
 .../storm/cluster/IStormClusterState.java       |  126 ++
 .../storm/cluster/StateStorageFactory.java      |   28 +
 .../storm/cluster/StormClusterStateImpl.java    |  720 ++++++++++
 .../org/apache/storm/cluster/VersionedData.java |   36 +
 .../apache/storm/cluster/ZKStateStorage.java    |  252 ++++
 .../org/apache/storm/command/HealthCheck.java   |  126 ++
 .../org/apache/storm/command/KillWorkers.java   |   38 +
 .../org/apache/storm/daemon/DaemonCommon.java   |   22 +
 .../storm/daemon/metrics/MetricsUtils.java      |   19 +-
 .../storm/daemon/supervisor/AdvancedFSOps.java  |  335 +++++
 .../storm/daemon/supervisor/BasicContainer.java |  638 +++++++++
 .../supervisor/BasicContainerLauncher.java      |   59 +
 .../storm/daemon/supervisor/Container.java      |  534 ++++++++
 .../daemon/supervisor/ContainerLauncher.java    |   96 ++
 .../supervisor/ContainerRecoveryException.java  |   29 +
 .../DefaultUncaughtExceptionHandler.java        |   31 +
 .../supervisor/EventManagerPushCallback.java    |   37 +
 .../daemon/supervisor/ExitCodeCallback.java     |   30 +
 .../storm/daemon/supervisor/Killable.java       |   50 +
 .../storm/daemon/supervisor/LocalContainer.java |   85 ++
 .../supervisor/LocalContainerLauncher.java      |   60 +
 .../daemon/supervisor/ReadClusterState.java     |  327 +++++
 .../daemon/supervisor/RunAsUserContainer.java   |   96 ++
 .../supervisor/RunAsUserContainerLauncher.java  |   57 +
 .../apache/storm/daemon/supervisor/Slot.java    |  785 +++++++++++
 .../daemon/supervisor/StandaloneSupervisor.java |   83 ++
 .../storm/daemon/supervisor/Supervisor.java     |  357 +++++
 .../daemon/supervisor/SupervisorUtils.java      |  311 +++++
 .../supervisor/timer/SupervisorHealthCheck.java |   41 +
 .../supervisor/timer/SupervisorHeartbeat.java   |   87 ++
 .../daemon/supervisor/timer/UpdateBlobs.java    |  105 ++
 .../org/apache/storm/event/EventManager.java    |   24 +
 .../org/apache/storm/event/EventManagerImp.java |   97 ++
 .../apache/storm/localizer/AsyncLocalizer.java  |  432 ++++++
 .../org/apache/storm/localizer/ILocalizer.java  |   70 +
 .../localizer/LocalDownloadedResource.java      |  146 ++
 .../LocalizedResourceRetentionSet.java          |    2 +-
 .../storm/localizer/LocalizedResourceSet.java   |    8 +-
 .../org/apache/storm/localizer/Localizer.java   |   38 +-
 .../storm/metric/StormMetricsRegistry.java      |   88 ++
 .../org/apache/storm/nimbus/ILeaderElector.java |    6 +-
 .../serialization/SerializationFactory.java     |   17 +-
 .../org/apache/storm/testing/FeederSpout.java   |   27 +-
 .../staticmocking/MockedSupervisorUtils.java    |   31 +
 .../apache/storm/trident/util/TridentUtils.java |    2 +-
 .../jvm/org/apache/storm/utils/ConfigUtils.java |  536 ++++++++
 .../org/apache/storm/utils/InprocMessaging.java |   69 +-
 .../jvm/org/apache/storm/utils/LocalState.java  |  112 +-
 .../org/apache/storm/utils/NimbusClient.java    |   19 +-
 .../src/jvm/org/apache/storm/utils/Time.java    |   51 +-
 .../src/jvm/org/apache/storm/utils/Utils.java   | 1054 +++++++++++++--
 .../storm/zookeeper/LeaderElectorImp.java       |  124 ++
 .../apache/storm/zookeeper/ZkEventTypes.java    |   43 +
 .../apache/storm/zookeeper/ZkKeeperStates.java  |   41 +
 .../org/apache/storm/zookeeper/Zookeeper.java   |  463 +++++++
 .../org/apache/storm/integration_test.clj       |   97 +-
 .../org/apache/storm/testing4j_test.clj         |   11 +-
 .../clj/org/apache/storm/logviewer_test.clj     |   55 +-
 .../test/clj/org/apache/storm/metrics_test.clj  |    9 +-
 .../test/clj/org/apache/storm/nimbus_test.clj   |   10 +-
 .../storm/security/auth/drpc_auth_test.clj      |   32 +-
 .../clj/org/apache/storm/supervisor_test.clj    |  751 -----------
 .../daemon/supervisor/BasicContainerTest.java   |  483 +++++++
 .../storm/daemon/supervisor/ContainerTest.java  |  265 ++++
 .../storm/daemon/supervisor/SlotTest.java       |  515 +++++++
 .../storm/localizer/AsyncLocalizerTest.java     |  187 +++
 .../LocalizedResourceRetentionSetTest.java      |   10 +-
 .../localizer/LocalizedResourceSetTest.java     |   12 +-
 .../apache/storm/localizer/LocalizerTest.java   |    9 +-
 91 files changed, 11457 insertions(+), 2521 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/bin/storm.py
----------------------------------------------------------------------
diff --git a/bin/storm.py b/bin/storm.py
index 61aae98..e221780 100755
--- a/bin/storm.py
+++ b/bin/storm.py
@@ -553,7 +553,7 @@ def healthcheck(*args):
     Run health checks on the local supervisor.
     """
     exec_storm_class(
-        "org.apache.storm.command.healthcheck",
+        "org.apache.storm.command.HealthCheck",
         args=args,
         jvmtype="-client",
         extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR, "bin")])
@@ -566,7 +566,7 @@ def kill_workers(*args):
     to have admin rights on the node to be able to successfully kill all workers.
     """
     exec_storm_class(
-        "org.apache.storm.command.kill_workers",
+        "org.apache.storm.command.KillWorkers",
         args=args,
         jvmtype="-client",
         extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR, "bin")])
@@ -650,7 +650,7 @@ def pacemaker(klass="org.apache.storm.pacemaker.pacemaker"):
         extrajars=cppaths,
         jvmopts=jvmopts)
 
-def supervisor(klass="org.apache.storm.daemon.supervisor"):
+def supervisor(klass="org.apache.storm.daemon.supervisor.Supervisor"):
     """Syntax: [storm supervisor]
 
     Launches the supervisor daemon. This command should be run

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/log4j2/cluster.xml
----------------------------------------------------------------------
diff --git a/log4j2/cluster.xml b/log4j2/cluster.xml
index e911823..a0ba3d1 100644
--- a/log4j2/cluster.xml
+++ b/log4j2/cluster.xml
@@ -18,7 +18,7 @@
 
 <configuration monitorInterval="60">
 <properties>
-    <property name="pattern">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} [%p] %msg%n</property>
+    <property name="pattern">%d{yyyy-MM-dd HH:mm:ss.SSS} %t %c{1.} [%p] %msg%n</property>
 </properties>
 <appenders>
     <RollingFile name="A1" immediateFlush="false"

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a1466c9..0665c10 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1067,8 +1067,8 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration>
-                    <source>1.7</source>
-                    <target>1.7</target>
+                    <source>1.8</source>
+                    <target>1.8</target>
                 </configuration>
             </plugin>
             <plugin>

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/cluster_state/zookeeper_state_factory.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/cluster_state/zookeeper_state_factory.clj b/storm-core/src/clj/org/apache/storm/cluster_state/zookeeper_state_factory.clj
index 061c9e8..fdef972 100644
--- a/storm-core/src/clj/org/apache/storm/cluster_state/zookeeper_state_factory.clj
+++ b/storm-core/src/clj/org/apache/storm/cluster_state/zookeeper_state_factory.clj
@@ -17,13 +17,16 @@
 (ns org.apache.storm.cluster-state.zookeeper-state-factory
   (:import [org.apache.curator.framework.state ConnectionStateListener])
   (:import [org.apache.zookeeper KeeperException$NoNodeException KeeperException$NodeExistsException]
-           [org.apache.storm.cluster ClusterState DaemonType])
+           [org.apache.storm.cluster ClusterState DaemonType ZKStateStorage])
   (:import [org.apache.storm.utils StormConnectionStateConverter])
   (:use [org.apache.storm cluster config log util])
   (:require [org.apache.storm [zookeeper :as zk]])
   (:gen-class
    :implements [org.apache.storm.cluster.ClusterStateFactory]))
 
+(defn -mkStore [this conf auth-conf, acls, context]
+  (ZKStateStorage. conf, auth-conf, acls, context))
+
 (defn -mkState [this conf auth-conf acls context]
   (let [zk (zk/mk-client conf (conf STORM-ZOOKEEPER-SERVERS) (conf STORM-ZOOKEEPER-PORT) :auth-conf auth-conf)]
     (zk/mkdirs zk (conf STORM-ZOOKEEPER-ROOT) acls)

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/command/kill_workers.clj b/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
deleted file mode 100644
index 2670735..0000000
--- a/storm-core/src/clj/org/apache/storm/command/kill_workers.clj
+++ /dev/null
@@ -1,33 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements.  See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership.  The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License.  You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.command.kill-workers
-  (:import [java.io File])
-  (:use [org.apache.storm.daemon common])
-  (:use [org.apache.storm util config])
-  (:require [org.apache.storm.daemon
-             [supervisor :as supervisor]])
-  (:gen-class))
-
-(defn -main 
-  "Construct the supervisor-data from scratch and kill the workers on this supervisor"
-  [& args]
-  (let [conf (read-storm-config)
-        conf (assoc conf STORM-LOCAL-DIR (. (File. (conf STORM-LOCAL-DIR)) getCanonicalPath))
-        isupervisor (supervisor/standalone-supervisor)
-        supervisor-data (supervisor/supervisor-data conf nil isupervisor)
-        ids (supervisor/my-worker-ids conf)]
-    (doseq [id ids]
-      (supervisor/shutdown-worker supervisor-data id))))

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/daemon/executor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/executor.clj b/storm-core/src/clj/org/apache/storm/daemon/executor.clj
index 2436048..3b20b84 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/executor.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/executor.clj
@@ -208,9 +208,9 @@
       (swap! interval-errors inc)
 
       (when (<= @interval-errors max-per-interval)
-        (cluster/report-error (:storm-cluster-state executor) (:storm-id executor) (:component-id executor)
+        (.report-error (:storm-cluster-state executor) (:storm-id executor) (:component-id executor)
                               (hostname storm-conf)
-                              (.getThisWorkerPort (:worker-context executor)) error)
+                              (.getThisWorkerPort ^WorkerTopologyContext (:worker-context executor)) error)
         ))))
 
 ;; in its own function so that it can be mocked out by tracked topologies

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
new file mode 100644
index 0000000..a02a8e2
--- /dev/null
+++ b/storm-core/src/clj/org/apache/storm/daemon/local_supervisor.clj
@@ -0,0 +1,28 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements.  See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership.  The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License.  You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+(ns org.apache.storm.daemon.local-supervisor
+  (:import [org.apache.storm.daemon.supervisor Supervisor]
+           [org.apache.storm.utils ConfigUtils])
+  (:use [org.apache.storm.daemon common])
+  (:gen-class))
+
+(defserverfn mk-local-supervisor [conf shared-context isupervisor]
+  (if (not (ConfigUtils/isLocalMode conf))
+    (throw
+      (IllegalArgumentException. "Cannot start server in distrubuted mode!")))
+  (let [supervisor-server (Supervisor. conf shared-context isupervisor)]
+    (.launch supervisor-server)
+    supervisor-server))

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
index 1ad88f9..92224c6 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
@@ -33,11 +33,12 @@
   (:import [java.nio ByteBuffer])
   (:import [org.apache.storm.utils Utils])
   (:import [org.apache.storm.daemon DirectoryCleaner])
+  (:import [org.apache.storm.daemon.supervisor SupervisorUtils])
   (:import [org.yaml.snakeyaml Yaml]
            [org.yaml.snakeyaml.constructor SafeConstructor])
   (:import [org.apache.storm.ui InvalidRequestException]
            [org.apache.storm.security.auth AuthUtils])
-  (:require [org.apache.storm.daemon common [supervisor :as supervisor]])
+  (:require [org.apache.storm.daemon common])
   (:require [compojure.route :as route]
             [compojure.handler :as handler]
             [ring.middleware.keyword-params]
@@ -157,10 +158,10 @@
 (defn get-alive-ids
   [conf now-secs]
   (->>
-    (supervisor/read-worker-heartbeats conf)
+    (clojurify-structure (SupervisorUtils/readWorkerHeartbeats conf))
     (remove
       #(or (not (val %))
-           (supervisor/is-worker-hb-timed-out? now-secs
+           (SupervisorUtils/isWorkerHbTimedOut now-secs
                                                (val %)
                                                conf)))
     keys

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj b/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
deleted file mode 100644
index 2888258..0000000
--- a/storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
+++ /dev/null
@@ -1,1272 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements.  See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership.  The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License.  You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.daemon.supervisor
-  (:import [java.io File IOException FileOutputStream])
-  (:import [org.apache.storm.scheduler ISupervisor]
-           [org.apache.storm.utils LocalState Time Utils]
-           [org.apache.storm.daemon Shutdownable]
-           [org.apache.storm Constants]
-           [org.apache.storm.cluster ClusterStateContext DaemonType]
-           [java.net JarURLConnection]
-           [java.net URI]
-           [org.apache.commons.io FileUtils])
-  (:use [org.apache.storm config util log timer local-state])
-  (:import [org.apache.storm.generated AuthorizationException KeyNotFoundException WorkerResources])
-  (:import [org.apache.storm.utils NimbusLeaderNotFoundException VersionInfo])
-  (:import [java.nio.file Files StandardCopyOption])
-  (:import [org.apache.storm Config])
-  (:import [org.apache.storm.generated WorkerResources ProfileAction StormTopology])
-  (:import [org.apache.storm.localizer LocalResource])
-  (:use [org.apache.storm.daemon common])
-  (:require [org.apache.storm.command [healthcheck :as healthcheck]])
-  (:require [org.apache.storm.daemon [worker :as worker]]
-            [org.apache.storm [process-simulator :as psim] [cluster :as cluster] [event :as event]]
-            [clojure.set :as set])
-  (:import [org.apache.thrift.transport TTransportException])
-  (:import [org.apache.zookeeper data.ACL ZooDefs$Ids ZooDefs$Perms])
-  (:import [org.yaml.snakeyaml Yaml]
-           [org.yaml.snakeyaml.constructor SafeConstructor])
-  (:require [metrics.gauges :refer [defgauge]])
-  (:require [metrics.meters :refer [defmeter mark!]])
-  (:gen-class
-    :methods [^{:static true} [launch [org.apache.storm.scheduler.ISupervisor] void]]))
-
-; declare to avoid triggering forward reference issue
-(declare rm-topo-files assigned-storm-ids-from-port-assignments)
-
-(defmeter supervisor:num-workers-launched)
-
-(defmulti download-storm-code cluster-mode)
-(defmulti launch-worker (fn [supervisor & _] (cluster-mode (:conf supervisor))))
-
-(def STORM-VERSION (VersionInfo/getVersion))
-
-(defprotocol SupervisorDaemon
-  (get-id [this])
-  (get-conf [this])
-  (shutdown-all-workers [this])
-  )
-
-(defn- assignments-snapshot [storm-cluster-state callback assignment-versions]
-  (let [storm-ids (.assignments storm-cluster-state callback)]
-    (let [new-assignments
-          (->>
-           (dofor [sid storm-ids]
-                  (let [recorded-version (:version (get assignment-versions sid))]
-                    (if-let [assignment-version (.assignment-version storm-cluster-state sid callback)]
-                      (if (= assignment-version recorded-version)
-                        {sid (get assignment-versions sid)}
-                        {sid (.assignment-info-with-version storm-cluster-state sid callback)})
-                      {sid nil})))
-           (apply merge)
-           (filter-val not-nil?))
-          new-profiler-actions
-          (->>
-            (dofor [sid (distinct storm-ids)]
-                   (if-let [topo-profile-actions (.get-topology-profile-requests storm-cluster-state sid false)]
-                      {sid topo-profile-actions}))
-           (apply merge))]
-         
-      {:assignments (into {} (for [[k v] new-assignments] [k (:data v)]))
-       :profiler-actions new-profiler-actions
-       :versions new-assignments})))
-
-(defn- read-my-executors [assignments-snapshot storm-id assignment-id]
-  (let [assignment (get assignments-snapshot storm-id)
-        my-slots-resources (into {}
-                                 (filter (fn [[[node _] _]] (= node assignment-id))
-                                         (:worker->resources assignment)))
-        my-executors (filter (fn [[_ [node _]]] (= node assignment-id))
-                             (:executor->node+port assignment))
-        port-executors (apply merge-with
-                              concat
-                              (for [[executor [_ port]] my-executors]
-                                {port [executor]}
-                                ))]
-    (into {} (for [[port executors] port-executors]
-               ;; need to cast to int b/c it might be a long (due to how yaml parses things)
-               ;; doall is to avoid serialization/deserialization problems with lazy seqs
-               [(Integer. port) (mk-local-assignment storm-id (doall executors) (get my-slots-resources [assignment-id port]))]
-               ))))
-
-(defn- read-assignments
-  "Returns map from port to struct containing :storm-id, :executors and :resources"
-  ([assignments-snapshot assignment-id]
-     (->> (dofor [sid (keys assignments-snapshot)] (read-my-executors assignments-snapshot sid assignment-id))
-          (apply merge-with (fn [& ignored] (throw-runtime "Should not have multiple topologies assigned to one port")))))
-  ([assignments-snapshot assignment-id existing-assignment retries]
-     (try (let [assignments (read-assignments assignments-snapshot assignment-id)]
-            (reset! retries 0)
-            assignments)
-          (catch RuntimeException e
-            (if (> @retries 2) (throw e) (swap! retries inc))
-            (log-warn (.getMessage e) ": retrying " @retries " of 3")
-            existing-assignment))))
-
-(defn- read-storm-code-locations
-  [assignments-snapshot]
-  (map-val :master-code-dir assignments-snapshot))
-
-(defn- read-downloaded-storm-ids [conf]
-  (map #(url-decode %) (read-dir-contents (supervisor-stormdist-root conf)))
-  )
-
-(defn read-worker-heartbeat [conf id]
-  (let [local-state (worker-state conf id)]
-    (try
-      (ls-worker-heartbeat local-state)
-      (catch Exception e
-        (log-warn e "Failed to read local heartbeat for workerId : " id ",Ignoring exception.")
-        nil))))
-
-
-(defn my-worker-ids [conf]
-  (read-dir-contents (worker-root conf)))
-
-(defn read-worker-heartbeats
-  "Returns map from worker id to heartbeat"
-  [conf]
-  (let [ids (my-worker-ids conf)]
-    (into {}
-      (dofor [id ids]
-        [id (read-worker-heartbeat conf id)]))
-    ))
-
-
-(defn matches-an-assignment? [worker-heartbeat assigned-executors]
-  (let [local-assignment (assigned-executors (:port worker-heartbeat))]
-    (and local-assignment
-         (= (:storm-id worker-heartbeat) (:storm-id local-assignment))
-         (= (disj (set (:executors worker-heartbeat)) Constants/SYSTEM_EXECUTOR_ID)
-            (set (:executors local-assignment))))))
-
-(let [dead-workers (atom #{})]
-  (defn get-dead-workers []
-    @dead-workers)
-  (defn add-dead-worker [worker]
-    (swap! dead-workers conj worker))
-  (defn remove-dead-worker [worker]
-    (swap! dead-workers disj worker)))
-
-(defn is-worker-hb-timed-out? [now hb conf]
-  (> (- now (:time-secs hb))
-     (conf SUPERVISOR-WORKER-TIMEOUT-SECS)))
-
-(defn read-allocated-workers
-  "Returns map from worker id to worker heartbeat. if the heartbeat is nil, then the worker is dead (timed out or never wrote heartbeat)"
-  [supervisor assigned-executors now]
-  (let [conf (:conf supervisor)
-        ^LocalState local-state (:local-state supervisor)
-        id->heartbeat (read-worker-heartbeats conf)
-        approved-ids (set (keys (ls-approved-workers local-state)))]
-    (into
-     {}
-     (dofor [[id hb] id->heartbeat]
-            (let [state (cond
-                         (not hb)
-                           :not-started
-                         (or (not (contains? approved-ids id))
-                             (not (matches-an-assignment? hb assigned-executors)))
-                           :disallowed
-                         (or
-                          (when (get (get-dead-workers) id)
-                            (log-message "Worker Process " id " has died!")
-                            true)
-                          (is-worker-hb-timed-out? now hb conf))
-                           :timed-out
-                         true
-                           :valid)]
-              (log-debug "Worker " id " is " state ": " (pr-str hb) " at supervisor time-secs " now)
-              [id [state hb]]
-              ))
-     )))
-
-(defn- wait-for-worker-launch [conf id start-time]
-  (let [state (worker-state conf id)]
-    (loop []
-      (let [hb (ls-worker-heartbeat state)]
-        (when (and
-               (not hb)
-               (<
-                (- (current-time-secs) start-time)
-                (conf SUPERVISOR-WORKER-START-TIMEOUT-SECS)
-                ))
-          (log-message id " still hasn't started")
-          (Time/sleep 500)
-          (recur)
-          )))
-    (when-not (ls-worker-heartbeat state)
-      (log-message "Worker " id " failed to start")
-      )))
-
-(defn- wait-for-workers-launch [conf ids]
-  (let [start-time (current-time-secs)]
-    (doseq [id ids]
-      (wait-for-worker-launch conf id start-time))
-    ))
-
-(defn generate-supervisor-id []
-  (uuid))
-
-(defnk worker-launcher [conf user args :environment {} :log-prefix nil :exit-code-callback nil :directory nil]
-  (let [_ (when (clojure.string/blank? user)
-            (throw (java.lang.IllegalArgumentException.
-                     "User cannot be blank when calling worker-launcher.")))
-        wl-initial (conf SUPERVISOR-WORKER-LAUNCHER)
-        storm-home (System/getProperty "storm.home")
-        wl (if wl-initial wl-initial (str storm-home "/bin/worker-launcher"))
-        command (concat [wl user] args)]
-    (log-message "Running as user:" user " command:" (pr-str command))
-    (launch-process command :environment environment :log-prefix log-prefix :exit-code-callback exit-code-callback :directory directory)
-  ))
-
-(defnk worker-launcher-and-wait [conf user args :environment {} :log-prefix nil]
-  (let [process (worker-launcher conf user args :environment environment)]
-    (if log-prefix
-      (read-and-log-stream log-prefix (.getInputStream process)))
-      (try
-        (.waitFor process)
-      (catch InterruptedException e
-        (log-message log-prefix " interrupted.")))
-      (.exitValue process)))
-
-(defn- rmr-as-user
-  "Launches a process owned by the given user that deletes the given path
-  recursively.  Throws RuntimeException if the directory is not removed."
-  [conf id path]
-  (let [user (Utils/getFileOwner path)]
-    (worker-launcher-and-wait conf
-      user
-      ["rmr" path]
-      :log-prefix (str "rmr " id))
-    (if (exists-file? path)
-      (throw (RuntimeException. (str path " was not deleted"))))))
-
-(defn try-cleanup-worker [conf id]
-  (try
-    (if (.exists (File. (worker-root conf id)))
-      (do
-        (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
-          (rmr-as-user conf id (worker-root conf id))
-          (do
-            (rmr (worker-heartbeats-root conf id))
-            ;; this avoids a race condition with worker or subprocess writing pid around same time
-            (rmr (worker-pids-root conf id))
-            (rmr (worker-root conf id))))
-        (remove-worker-user! conf id)
-        (remove-dead-worker id)
-      ))
-  (catch IOException e
-    (log-warn-error e "Failed to cleanup worker " id ". Will retry later"))
-  (catch RuntimeException e
-    (log-warn-error e "Failed to cleanup worker " id ". Will retry later"))))
-
-(defn shutdown-worker [supervisor id]
-  (log-message "Shutting down " (:supervisor-id supervisor) ":" id)
-  (let [conf (:conf supervisor)
-        pids (read-dir-contents (worker-pids-root conf id))
-        thread-pid (@(:worker-thread-pids-atom supervisor) id)
-        shutdown-sleep-secs (conf SUPERVISOR-WORKER-SHUTDOWN-SLEEP-SECS)
-        as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
-        user (get-worker-user conf id)]
-    (when thread-pid
-      (psim/kill-process thread-pid))
-    (doseq [pid pids]
-      (if as-user
-        (worker-launcher-and-wait conf user ["signal" pid "15"] :log-prefix (str "kill -15 " pid))
-        (kill-process-with-sig-term pid)))
-    (when-not (empty? pids)  
-      (log-message "Sleep " shutdown-sleep-secs " seconds for execution of cleanup threads on worker.")
-      (sleep-secs shutdown-sleep-secs))
-    (doseq [pid pids]
-      (if as-user
-        (worker-launcher-and-wait conf user ["signal" pid "9"] :log-prefix (str "kill -9 " pid))
-        (force-kill-process pid))
-      (if as-user
-        (try
-          (rmr-as-user conf id (worker-pid-path conf id pid))
-          (rmpath (worker-pid-path conf id pid))
-          (rmpath (worker-tmp-root conf id pid))
-          (catch Exception e
-            (log-warn-error e "Failed to cleanup pid dir: " pid " for worker " id". Will retry later")))))
-          ;; on windows, the supervisor may still holds the lock on the worker directory
-    (try-cleanup-worker conf id))
-  (log-message "Shut down " (:supervisor-id supervisor) ":" id))
-
-(def SUPERVISOR-ZK-ACLS
-  [(first ZooDefs$Ids/CREATOR_ALL_ACL)
-   (ACL. (bit-or ZooDefs$Perms/READ ZooDefs$Perms/CREATE) ZooDefs$Ids/ANYONE_ID_UNSAFE)])
-
-(defn supervisor-data [conf shared-context ^ISupervisor isupervisor]
-  {:conf conf
-   :shared-context shared-context
-   :isupervisor isupervisor
-   :active (atom true)
-   :uptime (uptime-computer)
-   :version STORM-VERSION
-   :worker-thread-pids-atom (atom {})
-   :storm-cluster-state (cluster/mk-storm-cluster-state conf :acls (when
-                                                                     (Utils/isZkAuthenticationConfiguredStormServer
-                                                                       conf)
-                                                                     SUPERVISOR-ZK-ACLS)
-                                                        :context (ClusterStateContext. DaemonType/SUPERVISOR))
-   :local-state (supervisor-state conf)
-   :supervisor-id (.getSupervisorId isupervisor)
-   :assignment-id (.getAssignmentId isupervisor)
-   :my-hostname (hostname conf)
-   :curr-assignment (atom nil) ;; used for reporting used ports when heartbeating
-   :heartbeat-timer (mk-timer :kill-fn (fn [t]
-                               (log-error t "Error when processing event")
-                               (exit-process! 20 "Error when processing an event")
-                               ))
-   :event-timer (mk-timer :kill-fn (fn [t]
-                                         (log-error t "Error when processing event")
-                                         (exit-process! 20 "Error when processing an event")
-                                         ))
-   :blob-update-timer (mk-timer :kill-fn (defn blob-update-timer
-                                           [t]
-                                           (log-error t "Error when processing event")
-                                           (exit-process! 20 "Error when processing a event"))
-                                :timer-name "blob-update-timer")
-   :localizer (Utils/createLocalizer conf (supervisor-local-dir conf))
-   :assignment-versions (atom {})
-   :sync-retry (atom 0)
-   :download-lock (Object.)
-   :stormid->profiler-actions (atom {})
-   })
-
-(defn required-topo-files-exist?
-  [conf storm-id]
-  (let [stormroot (supervisor-stormdist-root conf storm-id)
-        stormjarpath (supervisor-stormjar-path stormroot)
-        stormcodepath (supervisor-stormcode-path stormroot)
-        stormconfpath (supervisor-stormconf-path stormroot)]
-    (and (every? exists-file? [stormroot stormconfpath stormcodepath])
-         (or (local-mode? conf)
-             (exists-file? stormjarpath)))))
-
-(defn get-worker-assignment-helper-msg
-  [assignment supervisor port id]
-  (str (pr-str assignment) " for this supervisor " (:supervisor-id supervisor) " on port "
-    port " with id " id))
-
-(defn get-valid-new-worker-ids
-  [conf supervisor reassign-executors new-worker-ids]
-  (into {}
-    (remove nil?
-      (dofor [[port assignment] reassign-executors]
-        (let [id (new-worker-ids port)
-              storm-id (:storm-id assignment)
-              ^WorkerResources resources (:resources assignment)
-              mem-onheap (.get_mem_on_heap resources)]
-          ;; This condition checks for required files exist before launching the worker
-          (if (required-topo-files-exist? conf storm-id)
-            (do
-              (log-message "Launching worker with assignment "
-                (get-worker-assignment-helper-msg assignment supervisor port id))
-              (local-mkdirs (worker-pids-root conf id))
-              (local-mkdirs (worker-tmp-root conf id))
-              (local-mkdirs (worker-heartbeats-root conf id))
-              (launch-worker supervisor
-                (:storm-id assignment)
-                port
-                id
-                mem-onheap)
-              [id port])
-            (do
-              (log-message "Missing topology storm code, so can't launch worker with assignment "
-                (get-worker-assignment-helper-msg assignment supervisor port id))
-              nil)))))))
-
-(defn sync-processes [supervisor]
-  (let [conf (:conf supervisor)
-        ^LocalState local-state (:local-state supervisor)
-        assigned-executors (defaulted (ls-local-assignments local-state) {})
-        assigned-storm-ids (assigned-storm-ids-from-port-assignments assigned-executors)
-        localizer (:localizer supervisor)
-        now (current-time-secs)
-        allocated (read-allocated-workers supervisor assigned-executors now)
-        keepers (filter-val
-                 (fn [[state _]] (= state :valid))
-                 allocated)
-        keep-ports (set (for [[id [_ hb]] keepers] (:port hb)))
-        reassign-executors (select-keys-pred (complement keep-ports) assigned-executors)
-        new-worker-ids (into
-                        {}
-                        (for [port (keys reassign-executors)]
-                          [port (uuid)]))
-        all-downloaded-storm-ids (set (read-downloaded-storm-ids conf))]
-    ;; 1. to kill are those in allocated that are dead or disallowed
-    ;; 2. kill the ones that should be dead
-    ;;     - read pids, kill -9 and individually remove file
-    ;;     - rmr heartbeat dir, rmdir pid dir, rmdir id dir (catch exception and log)
-    ;; 3. remove any downloaded code that's no longer assigned to this supervisor
-    ;; 4. of the rest, figure out what assignments aren't yet satisfied
-    ;; 5. generate new worker ids, write new "approved workers" to LS
-    ;; 6. create local dir for worker id
-    ;; 7. launch new workers (give worker-id, port, and supervisor-id)
-    ;; 8. wait for workers launch
-    ;; Note: sync-processes runs based on local assignment, and avoid reading zk assignment.
-    ;; It's due to race condition between sync-supervisor and sync-processes.
-
-    (log-debug "Syncing processes")
-    (log-debug "Assigned executors: " assigned-executors)
-    (log-debug "Allocated: " allocated)
-    (doseq [[id [state heartbeat]] allocated]
-      (when (not= :valid state)
-        (log-message
-         "Shutting down and clearing state for id " id
-         ". Current supervisor time: " now
-         ". State: " state
-         ", Heartbeat: " (pr-str heartbeat))
-        (shutdown-worker supervisor id)))
-
-    (doseq [storm-id all-downloaded-storm-ids]
-      (when-not (assigned-storm-ids storm-id)
-        (log-message "Removing code for storm id "
-                     storm-id)
-        (rm-topo-files conf storm-id localizer true)))
-
-    (let [valid-new-worker-ids (get-valid-new-worker-ids conf supervisor reassign-executors new-worker-ids)]
-      (ls-approved-workers! local-state
-                        (merge
-                          (select-keys (ls-approved-workers local-state)
-                            (keys keepers))
-                          valid-new-worker-ids))
-      (wait-for-workers-launch conf (keys valid-new-worker-ids)))))
-
-(defn assigned-storm-ids-from-port-assignments [assignment]
-  (->> assignment
-       vals
-       (map :storm-id)
-       set))
-
-(defn get-blob-localname
-  "Given the blob information either gets the localname field if it exists,
-  else routines the default value passed in."
-  [blob-info defaultValue]
-  (or (get blob-info "localname") defaultValue))
-
-(defn should-uncompress-blob?
-  "Given the blob information returns the value of the uncompress field, handling it either being
-  a string or a boolean value, or if it's not specified then returns false"
-  [blob-info]
-  (Boolean. (get blob-info "uncompress")))
-
-(defn remove-blob-references
-  "Remove a reference to a blob when its no longer needed."
-  [localizer storm-id conf]
-  (let [storm-conf (read-supervisor-storm-conf conf storm-id)
-        blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)
-        user (storm-conf TOPOLOGY-SUBMITTER-USER)
-        topo-name (storm-conf TOPOLOGY-NAME)]
-    (if blobstore-map
-      (doseq [[k, v] blobstore-map]
-        (.removeBlobReference localizer
-          k
-          user
-          topo-name
-          (should-uncompress-blob? v))))))
-
-(defn blobstore-map-to-localresources
-  "Returns a list of LocalResources based on the blobstore-map passed in."
-  [blobstore-map]
-  (if blobstore-map
-    (for [[k, v] blobstore-map] (LocalResource. k (should-uncompress-blob? v)))
-    ()))
-
-(defn add-blob-references
-  "For each of the downloaded topologies, adds references to the blobs that the topologies are
-  using. This is used to reconstruct the cache on restart."
-  [localizer storm-id conf]
-  (let [storm-conf (read-supervisor-storm-conf conf storm-id)
-        blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)
-        user (storm-conf TOPOLOGY-SUBMITTER-USER)
-        topo-name (storm-conf TOPOLOGY-NAME)
-        localresources (blobstore-map-to-localresources blobstore-map)]
-    (if blobstore-map
-      (.addReferences localizer localresources user topo-name))))
-
-(defn rm-topo-files
-  [conf storm-id localizer rm-blob-refs?]
-  (let [path (supervisor-stormdist-root conf storm-id)]
-    (try
-      (if rm-blob-refs?
-        (remove-blob-references localizer storm-id conf))
-      (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
-        (rmr-as-user conf storm-id path)
-        (rmr (supervisor-stormdist-root conf storm-id)))
-      (catch Exception e
-        (log-message e (str "Exception removing: " storm-id))))))
-
-(defn verify-downloaded-files
-  "Check for the files exists to avoid supervisor crashing
-   Also makes sure there is no necessity for locking"
-  [conf localizer assigned-storm-ids all-downloaded-storm-ids]
-  (remove nil?
-    (into #{}
-      (for [storm-id all-downloaded-storm-ids
-            :when (contains? assigned-storm-ids storm-id)]
-        (when-not (required-topo-files-exist? conf storm-id)
-          (log-debug "Files not present in topology directory")
-          (rm-topo-files conf storm-id localizer false)
-          storm-id)))))
-
-(defn mk-synchronize-supervisor [supervisor sync-processes event-manager processes-event-manager]
-  (fn this []
-    (let [conf (:conf supervisor)
-          storm-cluster-state (:storm-cluster-state supervisor)
-          ^ISupervisor isupervisor (:isupervisor supervisor)
-          ^LocalState local-state (:local-state supervisor)
-          sync-callback (fn [& ignored] (.add event-manager this))
-          assignment-versions @(:assignment-versions supervisor)
-          {assignments-snapshot :assignments
-           storm-id->profiler-actions :profiler-actions
-           versions :versions}
-          (assignments-snapshot storm-cluster-state sync-callback assignment-versions)
-          storm-code-map (read-storm-code-locations assignments-snapshot)
-          all-downloaded-storm-ids (set (read-downloaded-storm-ids conf))
-          existing-assignment (ls-local-assignments local-state)
-          all-assignment (read-assignments assignments-snapshot
-                                           (:assignment-id supervisor)
-                                           existing-assignment
-                                           (:sync-retry supervisor))
-          new-assignment (->> all-assignment
-                              (filter-key #(.confirmAssigned isupervisor %)))
-          assigned-storm-ids (assigned-storm-ids-from-port-assignments new-assignment)
-          localizer (:localizer supervisor)
-          checked-downloaded-storm-ids (set (verify-downloaded-files conf localizer assigned-storm-ids all-downloaded-storm-ids))
-          downloaded-storm-ids (set/difference all-downloaded-storm-ids checked-downloaded-storm-ids)]
-
-      ;; 1. download assigned topology code if it's not downloaded
-      ;; 2. write new assignment to local assignment, and atom variables
-      ;; 3. trigger sync-processes to take care of the rest
-      ;; Note: mk-synchronize-supervisor runs based on zk assignment, and avoids touching local state
-      ;; except updating new assignment which sync-processes will refer.
-      ;; It's due to race condition between sync-supervisor and sync-processes.
-
-      (log-debug "Synchronizing supervisor")
-      (log-debug "Storm code map: " storm-code-map)
-      (log-debug "All assignment: " all-assignment)
-      (log-debug "New assignment: " new-assignment)
-      (log-debug "Assigned Storm Ids " assigned-storm-ids)
-      (log-debug "All Downloaded Ids " all-downloaded-storm-ids)
-      (log-debug "Checked Downloaded Ids " checked-downloaded-storm-ids)
-      (log-debug "Downloaded Ids " downloaded-storm-ids)
-      (log-debug "Storm Ids Profiler Actions " storm-id->profiler-actions)
-      ;; download code first
-      ;; This might take awhile
-      ;;   - should this be done separately from usual monitoring?
-      ;; should we only download when topology is assigned to this supervisor?
-      (doseq [[storm-id master-code-dir] storm-code-map]
-        (when (and (not (downloaded-storm-ids storm-id))
-                   (assigned-storm-ids storm-id))
-          (log-message "Downloading code for storm id " storm-id)
-          (try-cause
-            (download-storm-code conf storm-id master-code-dir localizer)
-
-            (catch NimbusLeaderNotFoundException e
-              (log-warn-error e "Nimbus leader was not available."))
-            (catch TTransportException e
-              (log-warn-error e "There was a connection problem with nimbus.")))
-          (log-message "Finished downloading code for storm id " storm-id)))
-
-      (log-debug "Writing new assignment "
-                 (pr-str new-assignment))
-      (doseq [p (set/difference (set (keys existing-assignment))
-                                (set (keys new-assignment)))]
-        (.killedWorker isupervisor (int p)))
-      (.assigned isupervisor (keys new-assignment))
-      (ls-local-assignments! local-state
-            new-assignment)
-      (reset! (:assignment-versions supervisor) versions)
-      (reset! (:stormid->profiler-actions supervisor) storm-id->profiler-actions)
-      (reset! (:curr-assignment supervisor) new-assignment)
-
-      (.add processes-event-manager sync-processes))))
-
-(defn mk-supervisor-capacities
-  [conf]
-  {Config/SUPERVISOR_MEMORY_CAPACITY_MB (double (conf SUPERVISOR-MEMORY-CAPACITY-MB))
-   Config/SUPERVISOR_CPU_CAPACITY (double (conf SUPERVISOR-CPU-CAPACITY))})
-
-(defn update-blobs-for-topology!
-  "Update each blob listed in the topology configuration if the latest version of the blob
-   has not been downloaded."
-  [conf storm-id localizer]
-  (let [storm-conf (read-supervisor-storm-conf conf storm-id)
-        blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)
-        user (storm-conf TOPOLOGY-SUBMITTER-USER)
-        localresources (blobstore-map-to-localresources blobstore-map)]
-    (try
-      (.updateBlobs localizer localresources user)
-      (catch AuthorizationException authExp
-        (log-error authExp))
-      (catch KeyNotFoundException knf
-        (log-error knf)))))
-
-(defn update-blobs-for-all-topologies-fn
-  "Returns a function that downloads all blobs listed in the topology configuration for all topologies assigned
-  to this supervisor, and creates version files with a suffix. The returned function is intended to be run periodically
-  by a timer, created elsewhere."
-  [supervisor]
-  (fn []
-    (try-cause
-      (let [conf (:conf supervisor)
-            downloaded-storm-ids (set (read-downloaded-storm-ids conf))
-            new-assignment @(:curr-assignment supervisor)
-            assigned-storm-ids (assigned-storm-ids-from-port-assignments new-assignment)]
-        (doseq [topology-id downloaded-storm-ids]
-          (let [storm-root (supervisor-stormdist-root conf topology-id)]
-            (when (assigned-storm-ids topology-id)
-              (log-debug "Checking Blob updates for storm topology id " topology-id " With target_dir: " storm-root)
-              (update-blobs-for-topology! conf topology-id (:localizer supervisor))))))
-      (catch TTransportException e
-        (log-error
-          e
-          "Network error while updating blobs, will retry again later"))
-      (catch NimbusLeaderNotFoundException e
-        (log-error
-          e
-          "Nimbus unavailable to update blobs, will retry again later")))))
-
-(defn jvm-cmd [cmd]
-  (let [java-home (.get (System/getenv) "JAVA_HOME")]
-    (if (nil? java-home)
-      cmd
-      (str java-home file-path-separator "bin" file-path-separator cmd))))
-
-(defn java-cmd []
-  (jvm-cmd "java"))
-
-(defn jmap-dump-cmd [profile-cmd pid target-dir]
-  [profile-cmd pid "jmap" target-dir])
-
-(defn jstack-dump-cmd [profile-cmd pid target-dir]
-  [profile-cmd pid "jstack" target-dir])
-
-(defn jprofile-start [profile-cmd pid]
-  [profile-cmd pid "start"])
-
-(defn jprofile-stop [profile-cmd pid target-dir]
-  [profile-cmd pid "stop" target-dir])
-
-(defn jprofile-dump [profile-cmd pid workers-artifacts-directory]
-  [profile-cmd pid "dump" workers-artifacts-directory])
-
-(defn jprofile-jvm-restart [profile-cmd pid]
-  [profile-cmd pid "kill"])
-
-(defn- delete-topology-profiler-action [storm-cluster-state storm-id profile-action]
-  (log-message "Deleting profiler action.." profile-action)
-  (.delete-topology-profile-requests storm-cluster-state storm-id profile-action))
-
-(defnk launch-profiler-action-for-worker
-  "Launch profiler action for a worker"
-  [conf user target-dir command :environment {} :exit-code-on-profile-action nil :log-prefix nil]
-  (if-let [run-worker-as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)]
-    (let [container-file (container-file-path target-dir)
-          script-file (script-file-path target-dir)]
-      (log-message "Running as user:" user " command:" (shell-cmd command))
-      (if (exists-file? container-file) (rmr-as-user conf container-file container-file))
-      (if (exists-file? script-file) (rmr-as-user conf script-file script-file))
-      (worker-launcher
-        conf
-        user
-        ["profiler" target-dir (write-script target-dir command :environment environment)]
-        :log-prefix log-prefix
-        :exit-code-callback exit-code-on-profile-action
-        :directory (File. target-dir)))
-    (launch-process
-      command
-      :environment environment
-      :log-prefix log-prefix
-      :exit-code-callback exit-code-on-profile-action
-      :directory (File. target-dir))))
-
-(defn mk-run-profiler-actions-for-all-topologies
-  "Returns a function that downloads all profile-actions listed for all topologies assigned
-  to this supervisor, executes those actions as user and deletes them from zookeeper."
-  [supervisor]
-  (fn []
-    (try
-      (let [conf (:conf supervisor)
-            stormid->profiler-actions @(:stormid->profiler-actions supervisor)
-            storm-cluster-state (:storm-cluster-state supervisor)
-            hostname (:my-hostname supervisor)
-            storm-home (System/getProperty "storm.home")
-            profile-cmd (str (clojure.java.io/file storm-home
-                                                   "bin"
-                                                   (conf WORKER-PROFILER-COMMAND)))]
-        (doseq [[storm-id profiler-actions] stormid->profiler-actions]
-          (when (not (empty? profiler-actions))
-            (doseq [pro-action profiler-actions]
-              (if (= hostname (:host pro-action))
-                (let [port (:port pro-action)
-                      action ^ProfileAction (:action pro-action)
-                      stop? (> (System/currentTimeMillis) (:timestamp pro-action))
-                      target-dir (worker-artifacts-root conf storm-id port)
-                      storm-conf (read-supervisor-storm-conf conf storm-id)
-                      user (storm-conf TOPOLOGY-SUBMITTER-USER)
-                      environment (if-let [env (storm-conf TOPOLOGY-ENVIRONMENT)] env {})
-                      worker-pid (slurp (worker-artifacts-pid-path conf storm-id port))
-                      log-prefix (str "ProfilerAction process " storm-id ":" port " PROFILER_ACTION: " action " ")
-                      ;; Until PROFILER_STOP action is invalid, keep launching profiler start in case worker restarted
-                      ;; The profiler plugin script validates if JVM is recording before starting another recording.
-                      command (cond
-                                (= action ProfileAction/JMAP_DUMP) (jmap-dump-cmd profile-cmd worker-pid target-dir)
-                                (= action ProfileAction/JSTACK_DUMP) (jstack-dump-cmd profile-cmd worker-pid target-dir)
-                                (= action ProfileAction/JPROFILE_DUMP) (jprofile-dump profile-cmd worker-pid target-dir)
-                                (= action ProfileAction/JVM_RESTART) (jprofile-jvm-restart profile-cmd worker-pid)
-                                (and (not stop?)
-                                     (= action ProfileAction/JPROFILE_STOP))
-                                  (jprofile-start profile-cmd worker-pid) ;; Ensure the profiler is still running
-                                (and stop? (= action ProfileAction/JPROFILE_STOP)) (jprofile-stop profile-cmd worker-pid target-dir))
-                      action-on-exit (fn [exit-code]
-                                       (log-message log-prefix " profile-action exited for code: " exit-code)
-                                       (if stop?
-                                         (delete-topology-profiler-action storm-cluster-state storm-id pro-action)))
-                      command (->> command (map str) (filter (complement empty?)))]
-
-                  (try
-                    (launch-profiler-action-for-worker conf
-                      user
-                      target-dir
-                      command
-                      :environment environment
-                      :exit-code-on-profile-action action-on-exit
-                      :log-prefix log-prefix)
-                    (catch IOException ioe
-                      (log-error ioe
-                        (str "Error in processing ProfilerAction '" action "' for " storm-id ":" port ", will retry later.")))
-                    (catch RuntimeException rte
-                      (log-error rte
-                        (str "Error in processing ProfilerAction '" action "' for " storm-id ":" port ", will retry later."))))))))))
-      (catch Exception e
-        (log-error e "Error running profiler actions, will retry again later")))))
-
-;; in local state, supervisor stores who its current assignments are
-;; another thread launches events to restart any dead processes if necessary
-(defserverfn mk-supervisor [conf shared-context ^ISupervisor isupervisor]
-  (log-message "Starting Supervisor with conf " conf)
-  (.prepare isupervisor conf (supervisor-isupervisor-dir conf))
-  (FileUtils/cleanDirectory (File. (supervisor-tmp-dir conf)))
-  (let [supervisor (supervisor-data conf shared-context isupervisor)
-        [event-manager processes-event-manager :as managers] [(event/event-manager false) (event/event-manager false)]
-        sync-processes (partial sync-processes supervisor)
-        synchronize-supervisor (mk-synchronize-supervisor supervisor sync-processes event-manager processes-event-manager)
-        synchronize-blobs-fn (update-blobs-for-all-topologies-fn supervisor)
-        downloaded-storm-ids (set (read-downloaded-storm-ids conf))
-        run-profiler-actions-fn (mk-run-profiler-actions-for-all-topologies supervisor)
-        heartbeat-fn (fn [] (.supervisor-heartbeat!
-                               (:storm-cluster-state supervisor)
-                               (:supervisor-id supervisor)
-                               (->SupervisorInfo (current-time-secs)
-                                                 (:my-hostname supervisor)
-                                                 (:assignment-id supervisor)
-                                                 (keys @(:curr-assignment supervisor))
-                                                  ;; used ports
-                                                 (.getMetadata isupervisor)
-                                                 (conf SUPERVISOR-SCHEDULER-META)
-                                                 ((:uptime supervisor))
-                                                 (:version supervisor)
-                                                 (mk-supervisor-capacities conf))))]
-    (heartbeat-fn)
-
-    ;; should synchronize supervisor so it doesn't launch anything after being down (optimization)
-    (schedule-recurring (:heartbeat-timer supervisor)
-                        0
-                        (conf SUPERVISOR-HEARTBEAT-FREQUENCY-SECS)
-                        heartbeat-fn)
-    (doseq [storm-id downloaded-storm-ids]
-      (add-blob-references (:localizer supervisor) storm-id
-        conf))
-    ;; do this after adding the references so we don't try to clean things being used
-    (.startCleaner (:localizer supervisor))
-
-    (when (conf SUPERVISOR-ENABLE)
-      ;; This isn't strictly necessary, but it doesn't hurt and ensures that the machine stays up
-      ;; to date even if callbacks don't all work exactly right
-      (schedule-recurring (:event-timer supervisor) 0 10 (fn [] (.add event-manager synchronize-supervisor)))
-      (schedule-recurring (:event-timer supervisor)
-                          0
-                          (conf SUPERVISOR-MONITOR-FREQUENCY-SECS)
-                          (fn [] (.add processes-event-manager sync-processes)))
-
-      ;; Blob update thread. Starts with 30 seconds delay, every 30 seconds
-      (schedule-recurring (:blob-update-timer supervisor)
-                          30
-                          30
-                          (fn [] (.add event-manager synchronize-blobs-fn)))
-
-      (schedule-recurring (:event-timer supervisor)
-                          (* 60 5)
-                          (* 60 5)
-                          (fn [] (let [health-code (healthcheck/health-check conf)
-                                       ids (my-worker-ids conf)]
-                                   (if (not (= health-code 0))
-                                     (do
-                                       (doseq [id ids]
-                                         (shutdown-worker supervisor id))
-                                       (throw (RuntimeException. "Supervisor failed health check. Exiting.")))))))
-
-      ;; Launch a thread that Runs profiler commands . Starts with 30 seconds delay, every 30 seconds
-      (schedule-recurring (:event-timer supervisor)
-                          30
-                          30
-                          (fn [] (.add event-manager run-profiler-actions-fn))))
-    (log-message "Starting supervisor with id " (:supervisor-id supervisor) " at host " (:my-hostname supervisor))
-    (reify
-     Shutdownable
-     (shutdown [this]
-               (log-message "Shutting down supervisor " (:supervisor-id supervisor))
-               (reset! (:active supervisor) false)
-               (cancel-timer (:heartbeat-timer supervisor))
-               (cancel-timer (:event-timer supervisor))
-               (cancel-timer (:blob-update-timer supervisor))
-               (.shutdown event-manager)
-               (.shutdown processes-event-manager)
-               (.shutdown (:localizer supervisor))
-               (.disconnect (:storm-cluster-state supervisor)))
-     SupervisorDaemon
-     (get-conf [this]
-       conf)
-     (get-id [this]
-       (:supervisor-id supervisor))
-     (shutdown-all-workers [this]
-       (let [ids (my-worker-ids conf)]
-         (doseq [id ids]
-           (shutdown-worker supervisor id)
-           )))
-     DaemonCommon
-     (waiting? [this]
-       (or (not @(:active supervisor))
-           (and
-            (timer-waiting? (:heartbeat-timer supervisor))
-            (timer-waiting? (:event-timer supervisor))
-            (every? (memfn waiting?) managers)))
-           ))))
-
-(defn kill-supervisor [supervisor]
-  (.shutdown supervisor)
-  )
-
-(defn setup-storm-code-dir
-  [conf storm-conf dir]
- (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
-  (worker-launcher-and-wait conf (storm-conf TOPOLOGY-SUBMITTER-USER) ["code-dir" dir] :log-prefix (str "setup conf for " dir))))
-
-(defn setup-blob-permission
-  [conf storm-conf path]
-  (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
-    (worker-launcher-and-wait conf (storm-conf TOPOLOGY-SUBMITTER-USER) ["blob" path] :log-prefix (str "setup blob permissions for " path))))
-
-(defn download-blobs-for-topology!
-  "Download all blobs passed by parameters for a given topology."
-  [conf storm-conf localizer tmproot blobs fn-local-resources fn-symlink-name]
-  (let [
-        user (storm-conf TOPOLOGY-SUBMITTER-USER)
-        topo-name (storm-conf TOPOLOGY-NAME)
-        user-dir (.getLocalUserFileCacheDir localizer user)
-        localresources (fn-local-resources blobs)]
-    (when localresources
-      (when-not (.exists user-dir)
-        (FileUtils/forceMkdir user-dir))
-      (try
-        (let [localized-resources (.getBlobs localizer localresources user topo-name user-dir)]
-          (setup-blob-permission conf storm-conf (.toString user-dir))
-          (doseq [local-rsrc localized-resources]
-            (let [rsrc-file-path (File. (.getFilePath local-rsrc))
-                  key-name (.getName rsrc-file-path)
-                  blob-symlink-target-name (.getName (File. (.getCurrentSymlinkPath local-rsrc)))
-                  symlink-name (fn-symlink-name blobs key-name)]
-              (create-symlink! tmproot (.getParent rsrc-file-path) symlink-name
-                               blob-symlink-target-name))))
-        (catch AuthorizationException authExp
-          (log-error authExp))
-        (catch KeyNotFoundException knf
-          (log-error knf))))))
-
-(defn download-blobs-in-blobstore-map-for-topology!
-  "Download all blobs listed in the topology configuration for a given topology."
-  [conf stormconf-path localizer tmproot]
-  (let [storm-conf (read-supervisor-storm-conf-given-path conf stormconf-path)
-        blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)]
-    (download-blobs-for-topology! conf storm-conf localizer tmproot blobstore-map
-                                  (fn [blobs] (blobstore-map-to-localresources blobs))
-                                  (fn [blobs key-name] (get-blob-localname (get blobs key-name) key-name)))))
-
-(defn download-dependencies-for-topology!
-  "Download all dependencies blobs listed in the topology configuration for a given topology."
-  [conf stormconf-path stormcode-path localizer tmproot]
-  (let [storm-conf (read-supervisor-storm-conf-given-path conf stormconf-path)
-        storm-code (read-supervisor-storm-code-given-path stormcode-path)
-        dependencies (concat (.get_dependency_jars ^StormTopology storm-code)
-                             (.get_dependency_artifacts ^StormTopology storm-code))]
-    (download-blobs-for-topology! conf storm-conf localizer tmproot dependencies
-                                  (fn [blobs] (map #(LocalResource. % false) blobs))
-                                  (fn [_ key-name] key-name))))
-
-(defn get-blob-file-names
-  [blobstore-map]
-  (if blobstore-map
-    (for [[k, data] blobstore-map]
-      (get-blob-localname data k))))
-
-(defn download-blobs-for-topology-succeed?
-  "Assert if all blobs are downloaded for the given topology"
-  [target-dir file-names]
-  (if-not (empty? file-names)
-    (every? #(Utils/checkFileExists target-dir %) file-names)
-    true))
-
-(defn download-blobs-in-blobstore-map-for-topology-succeed?
-  "Assert if all blobs in blobstore map are downloaded for the given topology"
-  [stormconf-path target-dir]
-  (let [storm-conf (clojurify-structure (Utils/fromCompressedJsonConf (FileUtils/readFileToByteArray (File. stormconf-path))))
-        blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)
-        file-names (get-blob-file-names blobstore-map)]
-    (download-blobs-for-topology-succeed? target-dir file-names)))
-
-(defn download-dependencies-for-topology-succeed?
-  "Assert if all dependencies blobs are downloaded for the given topology"
-  [stormcode-path target-dir]
-  (let [storm-code (read-supervisor-storm-code-given-path stormcode-path)
-        file-names (concat (.get_dependency_jars ^StormTopology storm-code)
-                           (.get_dependency_artifacts ^StormTopology storm-code))]
-    (download-blobs-for-topology-succeed? target-dir file-names)))
-
-;; distributed implementation
-(defmethod download-storm-code
-  :distributed [conf storm-id master-code-dir localizer]
-  ;; Downloading to permanent location is atomic
-  (let [tmproot (str (supervisor-tmp-dir conf) file-path-separator (uuid))
-        stormroot (supervisor-stormdist-root conf storm-id)
-        blobstore (Utils/getClientBlobStoreForSupervisor conf)]
-    (FileUtils/forceMkdir (File. tmproot))
-    (if-not on-windows?
-      (Utils/restrictPermissions tmproot)
-      (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
-        (throw-runtime (str "ERROR: Windows doesn't implement setting the correct permissions"))))
-    (Utils/downloadResourcesAsSupervisor (master-stormjar-key storm-id)
-      (supervisor-stormjar-path tmproot) blobstore)
-    (Utils/downloadResourcesAsSupervisor (master-stormcode-key storm-id)
-      (supervisor-stormcode-path tmproot) blobstore)
-    (Utils/downloadResourcesAsSupervisor (master-stormconf-key storm-id)
-      (supervisor-stormconf-path tmproot) blobstore)
-    (.shutdown blobstore)
-    (extract-dir-from-jar (supervisor-stormjar-path tmproot) RESOURCES-SUBDIR tmproot)
-    (download-blobs-in-blobstore-map-for-topology! conf (supervisor-stormconf-path tmproot) localizer
-      tmproot)
-    (download-dependencies-for-topology! conf (supervisor-stormconf-path tmproot) (supervisor-stormcode-path tmproot) localizer tmproot)
-    (if (and (download-blobs-in-blobstore-map-for-topology-succeed? (supervisor-stormconf-path tmproot) tmproot)
-             (download-dependencies-for-topology-succeed? (supervisor-stormcode-path tmproot) tmproot))
-      (do
-        (log-message "Successfully downloaded blob resources for storm-id " storm-id)
-        (if on-windows?
-          ; Files/move with non-empty directory doesn't work well on Windows
-          (FileUtils/moveDirectory (File. tmproot) (File. stormroot))
-          (do
-            (FileUtils/forceMkdir (File. stormroot))
-            (Files/move (.toPath (File. tmproot)) (.toPath (File. stormroot))
-                        (doto (make-array StandardCopyOption 1) (aset 0 StandardCopyOption/ATOMIC_MOVE)))))
-        (setup-storm-code-dir conf (read-supervisor-storm-conf conf storm-id) stormroot))
-      (do
-        (log-message "Failed to download blob resources for storm-id " storm-id)
-        (rmr tmproot)))))
-
-(defn write-log-metadata-to-yaml-file! [storm-id port data conf]
-  (let [file (get-log-metadata-file conf storm-id port)]
-    ;;run worker as user needs the directory to have special permissions
-    ;; or it is insecure
-    (when (not (.exists (.getParentFile file)))
-      (if (conf SUPERVISOR-RUN-WORKER-AS-USER)
-        (do (FileUtils/forceMkdir (.getParentFile file))
-            (setup-storm-code-dir conf (read-supervisor-storm-conf conf storm-id) (.getCanonicalPath (.getParentFile file))))
-        (.mkdirs (.getParentFile file))))
-    (let [writer (java.io.FileWriter. file)
-          yaml (Yaml.)]
-      (try
-        (.dump yaml data writer)
-        (finally
-          (.close writer))))))
-
-(defn write-log-metadata! [storm-conf user worker-id storm-id port conf]
-  (let [data {TOPOLOGY-SUBMITTER-USER user
-              "worker-id" worker-id
-              LOGS-GROUPS (sort (distinct (remove nil?
-                                           (concat
-                                             (storm-conf LOGS-GROUPS)
-                                             (storm-conf TOPOLOGY-GROUPS)))))
-              LOGS-USERS (sort (distinct (remove nil?
-                                           (concat
-                                             (storm-conf LOGS-USERS)
-                                             (storm-conf TOPOLOGY-USERS)))))}]
-    (write-log-metadata-to-yaml-file! storm-id port data conf)))
-
-(defn jlp [stormroot conf]
-  (let [resource-root (str stormroot File/separator RESOURCES-SUBDIR)
-        os (clojure.string/replace (System/getProperty "os.name") #"\s+" "_")
-        arch (System/getProperty "os.arch")
-        arch-resource-root (str resource-root File/separator os "-" arch)]
-    (str arch-resource-root File/pathSeparator resource-root File/pathSeparator (conf JAVA-LIBRARY-PATH))))
-
-(defn substitute-childopts
-  "Generates runtime childopts by replacing keys with topology-id, worker-id, port, mem-onheap"
-  [value worker-id topology-id port mem-onheap]
-  (let [replacement-map {"%ID%"          (str port)
-                         "%WORKER-ID%"   (str worker-id)
-                         "%TOPOLOGY-ID%"    (str topology-id)
-                         "%WORKER-PORT%" (str port)
-                         "%HEAP-MEM%" (str mem-onheap)}
-        sub-fn #(reduce (fn [string entry]
-                          (apply clojure.string/replace string entry))
-                        %
-                        replacement-map)]
-    (cond
-      (nil? value) nil
-      (sequential? value) (vec (map sub-fn value))
-      :else (-> value sub-fn (clojure.string/split #"\s+")))))
-
-
-(defn create-blobstore-links
-  "Create symlinks in worker launch directory for all blobs"
-  [conf storm-id worker-id]
-  (let [stormroot (supervisor-stormdist-root conf storm-id)
-        storm-conf (read-supervisor-storm-conf conf storm-id)
-        workerroot (worker-root conf worker-id)
-        blobstore-map (storm-conf TOPOLOGY-BLOBSTORE-MAP)
-        blob-file-names (get-blob-file-names blobstore-map)
-        resource-file-names (cons RESOURCES-SUBDIR blob-file-names)]
-    (log-message "Creating symlinks for worker-id: " worker-id " storm-id: "
-      storm-id " for files(" (count resource-file-names) "): " (pr-str resource-file-names))
-    (create-symlink! workerroot stormroot RESOURCES-SUBDIR)
-    (doseq [file-name blob-file-names]
-      (create-symlink! workerroot stormroot file-name file-name))))
-
-(defn create-artifacts-link
-  "Create a symlink from workder directory to its port artifacts directory"
-  [conf storm-id port worker-id]
-  (let [worker-dir (worker-root conf worker-id)
-        topo-dir (worker-artifacts-root conf storm-id)]
-    (log-message "Creating symlinks for worker-id: " worker-id " storm-id: "
-                 storm-id " to its port artifacts directory")
-    (if (.exists (File. worker-dir))
-      (create-symlink! worker-dir topo-dir "artifacts" port))))
-
-(defmethod launch-worker
-    :distributed [supervisor storm-id port worker-id mem-onheap]
-    (let [conf (:conf supervisor)
-          run-worker-as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
-          storm-home (System/getProperty "storm.home")
-          storm-options (System/getProperty "storm.options")
-          storm-conf-file (System/getProperty "storm.conf.file")
-          worker-tmp-dir (worker-tmp-root conf worker-id)
-          storm-log-dir LOG-DIR
-          storm-log-conf-dir (conf STORM-LOG4J2-CONF-DIR)
-          storm-log4j2-conf-dir (if storm-log-conf-dir
-                                  (if (is-absolute-path? storm-log-conf-dir)
-                                    storm-log-conf-dir
-                                    (str storm-home file-path-separator storm-log-conf-dir))
-                                  (str storm-home file-path-separator "log4j2"))
-          stormroot (supervisor-stormdist-root conf storm-id)
-          jlp (jlp stormroot conf)
-          stormjar (supervisor-stormjar-path stormroot)
-          storm-topology (read-supervisor-topology conf storm-id)
-          dependencies (concat (.get_dependency_jars ^StormTopology storm-topology)
-                               (.get_dependency_artifacts ^StormTopology storm-topology))
-          dependency-locations (map #(File. stormroot %) dependencies)
-          storm-conf (read-supervisor-storm-conf conf storm-id)
-          topo-classpath (if-let [cp (storm-conf TOPOLOGY-CLASSPATH)]
-                           [cp]
-                           [])
-          classpath (-> (worker-classpath)
-                        (add-to-classpath [stormjar])
-                        (add-to-classpath dependency-locations)
-                        (add-to-classpath topo-classpath))
-          top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS)
-          mem-onheap (if (and mem-onheap (> mem-onheap 0)) ;; not nil and not zero
-                       (int (Math/ceil mem-onheap)) ;; round up
-                       (storm-conf WORKER-HEAP-MEMORY-MB)) ;; otherwise use default value
-          gc-opts (substitute-childopts (if top-gc-opts top-gc-opts (conf WORKER-GC-CHILDOPTS)) worker-id storm-id port mem-onheap)
-          topo-worker-logwriter-childopts (storm-conf TOPOLOGY-WORKER-LOGWRITER-CHILDOPTS)
-          user (storm-conf TOPOLOGY-SUBMITTER-USER)
-          logfilename "worker.log"
-          workers-artifacts (worker-artifacts-root conf)
-          logging-sensitivity (storm-conf TOPOLOGY-LOGGING-SENSITIVITY "S3")
-          worker-childopts (when-let [s (conf WORKER-CHILDOPTS)]
-                             (substitute-childopts s worker-id storm-id port mem-onheap))
-          topo-worker-childopts (when-let [s (storm-conf TOPOLOGY-WORKER-CHILDOPTS)]
-                                  (substitute-childopts s worker-id storm-id port mem-onheap))
-          worker--profiler-childopts (if (conf WORKER-PROFILER-ENABLED)
-                                       (substitute-childopts (conf WORKER-PROFILER-CHILDOPTS) worker-id storm-id port mem-onheap)
-                                       "")
-          topology-worker-environment (if-let [env (storm-conf TOPOLOGY-ENVIRONMENT)]
-                                        (merge env {"LD_LIBRARY_PATH" jlp})
-                                        {"LD_LIBRARY_PATH" jlp})
-
-          log4j-configuration-file (str (if (.startsWith (System/getProperty "os.name") "Windows")
-                                          (if (.startsWith storm-log4j2-conf-dir "file:")
-                                            storm-log4j2-conf-dir
-                                            (str "file:///" storm-log4j2-conf-dir))
-                                          storm-log4j2-conf-dir)
-                                     file-path-separator "worker.xml")
-          command (concat
-                    [(java-cmd) "-cp" classpath 
-                     topo-worker-logwriter-childopts
-                     (str "-Dlogfile.name=" logfilename)
-                     (str "-Dstorm.home=" storm-home)
-                     (str "-Dworkers.artifacts=" workers-artifacts)
-                     (str "-Dstorm.id=" storm-id)
-                     (str "-Dworker.id=" worker-id)
-                     (str "-Dworker.port=" port)
-                     (str "-Dstorm.log.dir=" storm-log-dir)
-                     (str "-Dlog4j.configurationFile=" log4j-configuration-file)
-                     (str "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector")
-                     "org.apache.storm.LogWriter"]
-                    [(java-cmd) "-server"]
-                    worker-childopts
-                    topo-worker-childopts
-                    gc-opts
-                    worker--profiler-childopts
-                    [(str "-Djava.library.path=" jlp)
-                     (str "-Dlogfile.name=" logfilename)
-                     (str "-Dstorm.home=" storm-home)
-                     (str "-Dworkers.artifacts=" workers-artifacts)
-                     (str "-Dstorm.conf.file=" storm-conf-file)
-                     (str "-Dstorm.options=" storm-options)
-                     (str "-Dstorm.log.dir=" storm-log-dir)
-                     (str "-Djava.io.tmpdir=" worker-tmp-dir)
-                     (str "-Dlogging.sensitivity=" logging-sensitivity)
-                     (str "-Dlog4j.configurationFile=" log4j-configuration-file)
-                     (str "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector")
-                     (str "-Dstorm.id=" storm-id)
-                     (str "-Dworker.id=" worker-id)
-                     (str "-Dworker.port=" port)
-                     "-cp" classpath
-                     "org.apache.storm.daemon.worker"
-                     storm-id
-                     (:assignment-id supervisor)
-                     port
-                     worker-id])
-          command (->> command (map str) (filter (complement empty?)))]
-      (log-message "Launching worker with command: " (shell-cmd command))
-      (write-log-metadata! storm-conf user worker-id storm-id port conf)
-      (set-worker-user! conf worker-id user)
-      (create-artifacts-link conf storm-id port worker-id)
-      (let [log-prefix (str "Worker Process " worker-id)
-            callback (fn [exit-code]
-                       (log-message log-prefix " exited with code: " exit-code)
-                       (add-dead-worker worker-id))
-            worker-dir (worker-root conf worker-id)]
-        (remove-dead-worker worker-id)
-        (create-blobstore-links conf storm-id worker-id)
-        (if run-worker-as-user
-          (worker-launcher conf user ["worker" worker-dir (write-script worker-dir command :environment topology-worker-environment)] :log-prefix log-prefix :exit-code-callback callback :directory (File. worker-dir))
-          (launch-process command :environment topology-worker-environment :log-prefix log-prefix :exit-code-callback callback :directory (File. worker-dir)))
-        )))
-
-;; local implementation
-
-(defn resources-jar []
-  (->> (.split (current-classpath) File/pathSeparator)
-       (filter #(.endsWith  % ".jar"))
-       (filter #(zip-contains-dir? % RESOURCES-SUBDIR))
-       first ))
-
-(defmethod download-storm-code
-  :local [conf storm-id master-code-dir localizer]
-  (let [tmproot (str (supervisor-tmp-dir conf) file-path-separator (uuid))
-        stormroot (supervisor-stormdist-root conf storm-id)
-        blob-store (Utils/getNimbusBlobStore conf master-code-dir nil)]
-    (try
-      (FileUtils/forceMkdir (File. tmproot))
-      (with-open [fos-storm-code (FileOutputStream. (supervisor-stormcode-path tmproot))
-                  fos-storm-conf (FileOutputStream. (supervisor-stormconf-path tmproot))]
-        (.readBlobTo blob-store (master-stormcode-key storm-id) fos-storm-code nil)
-        (.readBlobTo blob-store (master-stormconf-key storm-id) fos-storm-conf nil))
-      (finally
-        (.shutdown blob-store)))
-    (FileUtils/moveDirectory (File. tmproot) (File. stormroot))
-    (setup-storm-code-dir conf (read-supervisor-storm-conf conf storm-id) stormroot)
-    (let [classloader (.getContextClassLoader (Thread/currentThread))
-          resources-jar (resources-jar)
-          url (.getResource classloader RESOURCES-SUBDIR)
-          target-dir (str stormroot file-path-separator RESOURCES-SUBDIR)]
-      (cond
-        resources-jar
-        (do
-          (log-message "Extracting resources from jar at " resources-jar " to " target-dir)
-          (extract-dir-from-jar resources-jar RESOURCES-SUBDIR stormroot))
-        url
-        (do
-          (log-message "Copying resources at " (str url) " to " target-dir)
-          (FileUtils/copyDirectory (File. (.getFile url)) (File. target-dir)))))))
-
-(defmethod launch-worker
-    :local [supervisor storm-id port worker-id mem-onheap]
-    (let [conf (:conf supervisor)
-          pid (uuid)
-          worker (worker/mk-worker conf
-                                   (:shared-context supervisor)
-                                   storm-id
-                                   (:assignment-id supervisor)
-                                   port
-                                   worker-id)]
-      (set-worker-user! conf worker-id "")
-      (psim/register-process pid worker)
-      (swap! (:worker-thread-pids-atom supervisor) assoc worker-id pid)
-      ))
-
-(defn -launch
-  [supervisor]
-  (log-message "Starting supervisor for storm version '" STORM-VERSION "'")
-  (let [conf (read-storm-config)]
-    (validate-distributed-mode! conf)
-    (let [supervisor (mk-supervisor conf nil supervisor)]
-      (add-shutdown-hook-with-force-kill-in-1-sec #(.shutdown supervisor)))
-    (defgauge supervisor:num-slots-used-gauge #(count (my-worker-ids conf)))
-    (start-metrics-reporters conf)))
-
-(defn standalone-supervisor []
-  (let [conf-atom (atom nil)
-        id-atom (atom nil)]
-    (reify ISupervisor
-      (prepare [this conf local-dir]
-        (reset! conf-atom conf)
-        (let [state (LocalState. local-dir)
-              curr-id (if-let [id (ls-supervisor-id state)]
-                        id
-                        (generate-supervisor-id))]
-          (ls-supervisor-id! state curr-id)
-          (reset! id-atom curr-id))
-        )
-      (confirmAssigned [this port]
-        true)
-      (getMetadata [this]
-        (doall (map int (get @conf-atom SUPERVISOR-SLOTS-PORTS))))
-      (getSupervisorId [this]
-        @id-atom)
-      (getAssignmentId [this]
-        @id-atom)
-      (killedWorker [this port]
-        )
-      (assigned [this ports]
-        ))))
-
-(defn -main []
-  (setup-default-uncaught-exception-handler)
-  (-launch (standalone-supervisor)))

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/daemon/worker.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/worker.clj b/storm-core/src/clj/org/apache/storm/daemon/worker.clj
index 1326672..62a36a2 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/worker.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/worker.clj
@@ -343,6 +343,7 @@
                                    (let [q-metrics (.getMetrics queue)]
                                      (/ (double (.population q-metrics)) (.capacity q-metrics))))
                                  short-executor-receive-queue-map)
+              local-pop (map-key int local-pop)
               remote-load (reduce merge (for [[np conn] @(:cached-node+port->socket worker)] (into {} (.getLoad conn remote-tasks))))
               now (System/currentTimeMillis)]
           (.setLocal load-mapping local-pop)

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/event.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/event.clj b/storm-core/src/clj/org/apache/storm/event.clj
deleted file mode 100644
index edc7616..0000000
--- a/storm-core/src/clj/org/apache/storm/event.clj
+++ /dev/null
@@ -1,71 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements.  See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership.  The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License.  You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-
-(ns org.apache.storm.event
-  (:use [org.apache.storm log util])
-  (:import [org.apache.storm.utils Time Utils])
-  (:import [java.io InterruptedIOException])
-  (:import [java.util.concurrent LinkedBlockingQueue TimeUnit]))
-
-(defprotocol EventManager
-  (add [this event-fn])
-  (waiting? [this])
-  (shutdown [this]))
-
-(defn event-manager
-  "Creates a thread to respond to events. Any error will cause process to halt"
-  [daemon?]
-  (let [added (atom 0)
-        processed (atom 0)
-        ^LinkedBlockingQueue queue (LinkedBlockingQueue.)
-        running (atom true)
-        runner (Thread.
-                 (fn []
-                   (try-cause
-                     (while @running
-                       (let [r (.take queue)]
-                         (r)
-                         (swap! processed inc)))
-                     (catch InterruptedIOException t
-                       (log-message "Event manager interrupted while doing IO"))
-                     (catch InterruptedException t
-                       (log-message "Event manager interrupted"))
-                     (catch Throwable t
-                       (log-error t "Error when processing event")
-                       (exit-process! 20 "Error when processing an event")))))]
-    (.setDaemon runner daemon?)
-    (.start runner)
-    (reify
-      EventManager
-
-      (add
-        [this event-fn]
-        ;; should keep track of total added and processed to know if this is finished yet
-        (when-not @running
-          (throw (RuntimeException. "Cannot add events to a shutdown event manager")))
-        (swap! added inc)
-        (.put queue event-fn))
-
-      (waiting?
-        [this]
-        (or (Time/isThreadWaiting runner)
-            (= @processed @added)))
-
-      (shutdown
-        [this]
-        (reset! running false)
-        (.interrupt runner)
-        (.join runner)))))

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
index cede59e..077d528 100644
--- a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
+++ b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
@@ -26,7 +26,7 @@
             HBExecutionException HBServerMessageType HBMessage
             HBMessageData HBPulse]
            [org.apache.storm.cluster_state zookeeper_state_factory]
-           [org.apache.storm.cluster ClusterState]
+           [org.apache.storm.cluster ClusterState ZKStateStorage]
            [org.apache.storm.pacemaker PacemakerClient])
   (:gen-class
    :implements [org.apache.storm.cluster.ClusterStateFactory]))
@@ -38,6 +38,10 @@
 (defn makeZKState [conf auth-conf acls context]
   (.mkState (zookeeper_state_factory.) conf auth-conf acls context))
 
+(defn -mkStore [this conf auth-conf, acls, context]
+  ;; This is only used by the supervisor (this does not support pacemaker)!!!
+  (ZKStateStorage. conf, auth-conf, acls, context))
+
 (def max-retries 10)
 
 (defn -mkState [this conf auth-conf acls context]


[15/21] storm git commit: Bad include for JDK 7 support

Posted by ka...@apache.org.
Bad include for JDK 7 support


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/905c4436
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/905c4436
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/905c4436

Branch: refs/heads/1.x-branch
Commit: 905c443653f579cd0af96d1c799843a1b718801f
Parents: b12c288
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Sep 26 09:21:32 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Sep 26 09:21:32 2016 -0500

----------------------------------------------------------------------
 .../src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java  | 1 -
 1 file changed, 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/905c4436/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
index f1140f7..977cb2b 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -30,7 +30,6 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.stream.Collectors;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.storm.Config;


[14/21] storm git commit: Merge branch 'STORM-2122' of https://github.com/revans2/incubator-storm into STORM-2122

Posted by ka...@apache.org.
Merge branch 'STORM-2122' of https://github.com/revans2/incubator-storm into STORM-2122


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b12c288d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b12c288d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b12c288d

Branch: refs/heads/1.x-branch
Commit: b12c288dcf04c3a89d8759fa56d836d8371d20cf
Parents: 2297757
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Sun Sep 25 20:57:04 2016 +0900
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Sep 26 08:23:36 2016 -0500

----------------------------------------------------------------------
 .../storm/daemon/supervisor/BasicContainer.java | 93 +++++++++++++++++---
 1 file changed, 79 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/b12c288d/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
index b1fa226..f1140f7 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -23,8 +23,9 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -470,6 +471,82 @@ public class BasicContainer extends Container {
         return log4jConfigurationDir + Utils.FILE_PATH_SEPARATOR + "worker.xml";
     }
     
+    private static class DependencyLocations {
+        private List<String> _data = null;
+        private final Map<String, Object> _conf;
+        private final String _topologyId;
+        private final AdvancedFSOps _ops;
+        private final String _stormRoot;
+        
+        public DependencyLocations(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, final String stormRoot) {
+            _conf = conf;
+            _topologyId = topologyId;
+            _ops = ops;
+            _stormRoot = stormRoot;
+        }
+        
+        public String toString() {
+            List<String> data;
+            synchronized(this) {
+                data = _data;
+            }
+            return "DEP_LOCS for " + _topologyId +" => " + data;
+        }
+        
+        public synchronized List<String> get() throws IOException {
+            if (_data != null) {
+                return _data;
+            }
+            final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _ops);
+            final List<String> dependencyLocations = new ArrayList<>();
+            if (stormTopology.get_dependency_jars() != null) {
+                for (String dependency : stormTopology.get_dependency_jars()) {
+                    dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath());
+                }
+            }
+
+            if (stormTopology.get_dependency_artifacts() != null) {
+                for (String dependency : stormTopology.get_dependency_artifacts()) {
+                    dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath());
+                }
+            }
+            _data = dependencyLocations;
+            return _data;
+        }
+    }
+
+    static class DepLRUCache {
+        public final int _maxSize = 100; //We could make this configurable in the future...
+        
+        @SuppressWarnings("serial")
+        private LinkedHashMap<String, DependencyLocations> _cache = new LinkedHashMap<String, DependencyLocations>() {
+            @Override
+            protected boolean removeEldestEntry(Map.Entry<String,DependencyLocations> eldest) {
+                return (size() > _maxSize);
+            }
+        };
+        
+        public synchronized DependencyLocations get(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, String stormRoot) {
+            //Only go off of the topology id for now.
+            DependencyLocations dl = _cache.get(topologyId);
+            if (dl == null) {
+                _cache.putIfAbsent(topologyId, new DependencyLocations(conf, topologyId, ops, stormRoot));
+                dl = _cache.get(topologyId);
+            }
+            return dl;
+        }
+        
+        public synchronized void clear() {
+            _cache.clear();
+        }
+    }
+    
+    static final DepLRUCache DEP_LOC_CACHE = new DepLRUCache();
+    
+    public static List<String> getDependencyLocationsFor(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, String stormRoot) throws IOException {
+        return DEP_LOC_CACHE.get(conf, topologyId, ops, stormRoot).get();
+    }
+    
     /**
      * Get parameters for the class path of the worker process.  Also used by the
      * log Writer
@@ -479,19 +556,7 @@ public class BasicContainer extends Container {
      */
     private List<String> getClassPathParams(final String stormRoot) throws IOException {
         final String stormJar = ConfigUtils.supervisorStormJarPath(stormRoot);
-        final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _ops);
-        final List<String> dependencyLocations = new ArrayList<>();
-        if (stormTopology.get_dependency_jars() != null) {
-            for (String dependency : stormTopology.get_dependency_jars()) {
-                dependencyLocations.add(new File(stormRoot, dependency).getAbsolutePath());
-            }
-        }
-
-        if (stormTopology.get_dependency_artifacts() != null) {
-            for (String dependency : stormTopology.get_dependency_artifacts()) {
-                dependencyLocations.add(new File(stormRoot, dependency).getAbsolutePath());
-            }
-        }
+        final List<String> dependencyLocations = getDependencyLocationsFor(_conf, _topologyId, _ops, stormRoot);
         final String workerClassPath = getWorkerClassPath(stormJar, dependencyLocations);
         
         List<String> classPathParams = new ArrayList<>();


[06/21] storm git commit: STORM-2018: Supervisor V2

Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerLauncher.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerLauncher.java
new file mode 100644
index 0000000..bfba2ee
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerLauncher.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.storm.Config;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.messaging.IContext;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Launches containers
+ */
+public abstract class ContainerLauncher {
+    private static final Logger LOG = LoggerFactory.getLogger(ContainerLauncher.class);
+    
+    /**
+     * Factory to create the right container launcher 
+     * for the config and the environment.
+     * @param conf the config
+     * @param supervisorId the ID of the supervisor
+     * @param sharedContext Used in local mode to let workers talk together without netty
+     * @return the proper container launcher
+     * @throws IOException on any error
+     */
+    public static ContainerLauncher make(Map<String, Object> conf, String supervisorId, IContext sharedContext) throws IOException {
+        if (ConfigUtils.isLocalMode(conf)) {
+            return new LocalContainerLauncher(conf, supervisorId, sharedContext);
+        }
+        
+        if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
+            return new RunAsUserContainerLauncher(conf, supervisorId);
+        }
+        return new BasicContainerLauncher(conf, supervisorId);
+    }
+    
+    protected ContainerLauncher() {
+        //Empty
+    }
+
+    /**
+     * Launch a container in a given slot
+     * @param port the port to run this on
+     * @param assignment what to launch
+     * @param state the current state of the supervisor
+     * @return The container that can be used to manager the processes.
+     * @throws IOException on any error 
+     */
+    public abstract Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException;
+    
+    /**
+     * Recover a container for a running process
+     * @param port the port the assignment is running on
+     * @param assignment the assignment that was launched
+     * @param state the current state of the supervisor
+     * @return The container that can be used to manage the processes.
+     * @throws IOException on any error
+     * @throws ContainerRecoveryException if the Container could not be recovered
+     */
+    public abstract Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException, ContainerRecoveryException;
+    
+    /**
+     * Try to recover a container using just the worker ID.  
+     * The result is really only useful for killing the container
+     * and so is returning a Killable.  Even if a Container is returned
+     * do not case the result to Container because only the Killable APIs
+     * are guaranteed to work. 
+     * @param workerId the id of the worker to use
+     * @param localState the state of the running supervisor
+     * @return a Killable that can be used to kill the underlying container.
+     * @throws IOException on any error
+     * @throws ContainerRecoveryException if the Container could not be recovered
+     */
+    public abstract Killable recoverContainer(String workerId, LocalState localState) throws IOException, ContainerRecoveryException;
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java
new file mode 100644
index 0000000..7ab6e67
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ContainerRecoveryException.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+/**
+ * Could not recover the container.
+ */
+public class ContainerRecoveryException extends RuntimeException {
+
+    public ContainerRecoveryException(String message) {
+        super(message);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
new file mode 100644
index 0000000..8785f86
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/DefaultUncaughtExceptionHandler.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DefaultUncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
+    private static final Logger LOG = LoggerFactory.getLogger(DefaultUncaughtExceptionHandler.class);
+    @Override
+    public void uncaughtException(Thread t, Throwable e) {
+        LOG.error("Error when processing event", e);
+        Utils.exitProcess(20, "Error when processing an event");
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java
new file mode 100644
index 0000000..177bf67
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/EventManagerPushCallback.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.storm.event.EventManager;
+
+public class EventManagerPushCallback implements Runnable {
+
+    private EventManager eventManager;
+
+    private Runnable cb;
+
+    public EventManagerPushCallback(Runnable cb, EventManager eventManager) {
+        this.eventManager = eventManager;
+        this.cb = cb;
+    }
+
+    @Override
+    public void run() {
+        eventManager.add(cb);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java
new file mode 100644
index 0000000..082f205
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ExitCodeCallback.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+/**
+ * A callback that can accept an integer.
+ */
+public interface ExitCodeCallback {
+    
+    /**
+     * The process finished 
+     * @param exitCode the exit code of the finished process.
+     */
+    public void call(int exitCode);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Killable.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Killable.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Killable.java
new file mode 100644
index 0000000..8d6d8e0
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Killable.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+
+public interface Killable {
+    
+    /**
+     * Kill the processes in this container nicely.
+     * kill -15 equivalent
+     * @throws IOException on any error
+     */
+    public void kill() throws IOException;
+    
+    /**
+     * Kill the processes in this container violently.
+     * kill -9 equivalent
+     * @throws IOException on any error
+     */
+    public void forceKill() throws IOException;
+    
+    /**
+     * @return true if all of the processes are dead, else false
+     * @throws IOException on any error
+     */
+    public boolean areAllProcessesDead() throws IOException;
+    
+    /**
+     * Clean up the container. It is not coming back.
+     * by default do the same thing as when restarting.
+     * @throws IOException on any error
+     */
+    public void cleanUp() throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java
new file mode 100644
index 0000000..f21efa4
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainer.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.storm.ProcessSimulator;
+import org.apache.storm.daemon.Shutdownable;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.messaging.IContext;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import clojure.java.api.Clojure;
+import clojure.lang.IFn;
+
+public class LocalContainer extends Container {
+    private static final Logger LOG = LoggerFactory.getLogger(LocalContainer.class);
+    private volatile boolean _isAlive = false;
+    private final IContext _sharedContext;
+    
+    public LocalContainer(Map<String, Object> conf, String supervisorId, int port, LocalAssignment assignment, IContext sharedContext) throws IOException {
+        super(ContainerType.LAUNCH, conf, supervisorId, port, assignment, null, null, null);
+        _sharedContext = sharedContext;
+        _workerId = Utils.uuid();
+    }
+    
+    @Override
+    public void launch() throws IOException {
+        //TODO when worker goes to java, just call it directly (not through clojure)
+        IFn mkWorker = Clojure.var("org.apache.storm.daemon.worker", "mk-worker");
+
+        Shutdownable worker = (Shutdownable) mkWorker.invoke(_conf, _sharedContext, _topologyId, _supervisorId, _port, _workerId);
+        saveWorkerUser(System.getProperty("user.name"));
+        ProcessSimulator.registerProcess(_workerId, worker);
+        _isAlive = true;
+    }
+
+    @Override
+    public void kill() throws IOException {
+        ProcessSimulator.killProcess(_workerId);
+        _isAlive = false;
+        //Make sure the worker is down before we try to shoot any child processes
+        super.kill();
+    }
+
+    @Override
+    public boolean areAllProcessesDead() throws IOException {
+        return !_isAlive && super.areAllProcessesDead();
+    }
+
+    @Override
+    public void relaunch() throws IOException {
+        LOG.warn("NOOP relaunch in local mode...");
+    }
+
+    @Override
+    public boolean didMainProcessExit() {
+        //In local mode the main process should never exit on it's own
+        return false;
+    }
+
+    @Override
+    public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException {
+        throw new RuntimeException("Profiling requests are not supported in local mode");
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java
new file mode 100644
index 0000000..c25bc49
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/LocalContainerLauncher.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.messaging.IContext;
+import org.apache.storm.utils.LocalState;
+
+/**
+ * Launch Containers in local mode.
+ */
+public class LocalContainerLauncher extends ContainerLauncher {
+    private final Map<String, Object> _conf;
+    private final String _supervisorId;
+    private final IContext _sharedContext;
+
+    public LocalContainerLauncher(Map<String, Object> conf, String supervisorId, IContext sharedContext) {
+        _conf = conf;
+        _supervisorId = supervisorId;
+        _sharedContext = sharedContext;
+    }
+
+    @Override
+    public Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+        LocalContainer ret = new LocalContainer(_conf, _supervisorId, port, assignment, _sharedContext);
+        ret.setup();
+        ret.launch();
+        return ret;
+    }
+
+    @Override
+    public Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+        //We are in the same process we cannot recover anything
+        throw new ContainerRecoveryException("Local Mode Recovery is not supported");
+    }
+
+    @Override
+    public Killable recoverContainer(String workerId, LocalState localState) throws IOException {
+        //We are in the same process we cannot recover anything
+        throw new ContainerRecoveryException("Local Mode Recovery is not supported");
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
new file mode 100644
index 0000000..27f18b6
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
@@ -0,0 +1,327 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.storm.Config;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.cluster.VersionedData;
+import org.apache.storm.daemon.supervisor.Slot.MachineState;
+import org.apache.storm.daemon.supervisor.Slot.TopoProfileAction;
+import org.apache.storm.event.EventManager;
+import org.apache.storm.generated.Assignment;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.NodeInfo;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.generated.WorkerResources;
+import org.apache.storm.localizer.ILocalizer;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ReadClusterState implements Runnable, AutoCloseable {
+    private static final Logger LOG = LoggerFactory.getLogger(ReadClusterState.class);
+    
+    private final Map<String, Object> superConf;
+    private final IStormClusterState stormClusterState;
+    private final EventManager syncSupEventManager;
+    private final AtomicReference<Map<String, VersionedData<Assignment>>> assignmentVersions;
+    private final Map<Integer, Slot> slots = new HashMap<>();
+    private final AtomicInteger readRetry = new AtomicInteger(0);
+    private final String assignmentId;
+    private final ISupervisor iSuper;
+    private final ILocalizer localizer;
+    private final ContainerLauncher launcher;
+    private final String host;
+    private final LocalState localState;
+    private final IStormClusterState clusterState;
+    private final AtomicReference<Map<Long, LocalAssignment>> cachedAssignments;
+    
+    public ReadClusterState(Supervisor supervisor) throws Exception {
+        this.superConf = supervisor.getConf();
+        this.stormClusterState = supervisor.getStormClusterState();
+        this.syncSupEventManager = supervisor.getEventManger();
+        this.assignmentVersions = new AtomicReference<>(new HashMap<>());
+        this.assignmentId = supervisor.getAssignmentId();
+        this.iSuper = supervisor.getiSupervisor();
+        this.localizer = supervisor.getAsyncLocalizer();
+        this.host = supervisor.getHostName();
+        this.localState = supervisor.getLocalState();
+        this.clusterState = supervisor.getStormClusterState();
+        this.cachedAssignments = supervisor.getCurrAssignment();
+        
+        this.launcher = ContainerLauncher.make(superConf, assignmentId, supervisor.getSharedContext());
+        
+        @SuppressWarnings("unchecked")
+        List<Number> ports = (List<Number>)superConf.get(Config.SUPERVISOR_SLOTS_PORTS);
+        for (Number port: ports) {
+            slots.put(port.intValue(), mkSlot(port.intValue()));
+        }
+        
+        try {
+            Collection<String> workers = SupervisorUtils.supervisorWorkerIds(superConf);
+            for (Slot slot: slots.values()) {
+                String workerId = slot.getWorkerId();
+                if (workerId != null) {
+                    workers.remove(workerId);
+                }
+            }
+            if (!workers.isEmpty()) {
+                supervisor.killWorkers(workers, launcher);
+            }
+        } catch (Exception e) {
+            LOG.warn("Error trying to clean up old workers", e);
+        }
+
+        //All the slots/assignments should be recovered now, so we can clean up anything that we don't expect to be here
+        try {
+            localizer.cleanupUnusedTopologies();
+        } catch (Exception e) {
+            LOG.warn("Error trying to clean up old topologies", e);
+        }
+        
+        for (Slot slot: slots.values()) {
+            slot.start();
+        }
+    }
+
+    private Slot mkSlot(int port) throws Exception {
+        return new Slot(localizer, superConf, launcher, host, port,
+                localState, clusterState, iSuper, cachedAssignments);
+    }
+    
+    @Override
+    public synchronized void run() {
+        try {
+            Runnable syncCallback = new EventManagerPushCallback(this, syncSupEventManager);
+            List<String> stormIds = stormClusterState.assignments(syncCallback);
+            Map<String, VersionedData<Assignment>> assignmentsSnapshot =
+                    getAssignmentsSnapshot(stormClusterState, stormIds, assignmentVersions.get(), syncCallback);
+            
+            Map<Integer, LocalAssignment> allAssignments =
+                    readAssignments(assignmentsSnapshot);
+            if (allAssignments == null) {
+                //Something odd happened try again later
+                return;
+            }
+            Map<String, List<ProfileRequest>> topoIdToProfilerActions = getProfileActions(stormClusterState, stormIds);
+            
+            HashSet<Integer> assignedPorts = new HashSet<>();
+            LOG.debug("Synchronizing supervisor");
+            LOG.debug("All assignment: {}", allAssignments);
+            LOG.debug("Topology Ids -> Profiler Actions {}", topoIdToProfilerActions);
+            for (Integer port: allAssignments.keySet()) {
+                if (iSuper.confirmAssigned(port)) {
+                    assignedPorts.add(port);
+                }
+            }
+            HashSet<Integer> allPorts = new HashSet<>(assignedPorts);
+            allPorts.addAll(slots.keySet());
+            
+            Map<Integer, Set<TopoProfileAction>> filtered = new HashMap<>();
+            for (Entry<String, List<ProfileRequest>> entry: topoIdToProfilerActions.entrySet()) {
+                String topoId = entry.getKey();
+                if (entry.getValue() != null) {
+                    for (ProfileRequest req: entry.getValue()) {
+                        NodeInfo ni = req.get_nodeInfo();
+                        if (host.equals(ni.get_node())) {
+                            Long port = ni.get_port().iterator().next();
+                            Set<TopoProfileAction> actions = filtered.get(port);
+                            if (actions == null) {
+                                actions = new HashSet<>();
+                                filtered.put(port.intValue(), actions);
+                            }
+                            actions.add(new TopoProfileAction(topoId, req));
+                        }
+                    }
+                }
+            }
+            
+            for (Integer port: allPorts) {
+                Slot slot = slots.get(port);
+                if (slot == null) {
+                    slot = mkSlot(port);
+                    slots.put(port, slot);
+                    slot.start();
+                }
+                slot.setNewAssignment(allAssignments.get(port));
+                slot.addProfilerActions(filtered.get(port));
+            }
+            
+        } catch (Exception e) {
+            LOG.error("Failed to Sync Supervisor", e);
+            throw new RuntimeException(e);
+        }
+    }
+    
+    protected Map<String, VersionedData<Assignment>> getAssignmentsSnapshot(IStormClusterState stormClusterState, List<String> topoIds,
+            Map<String, VersionedData<Assignment>> localAssignmentVersion, Runnable callback) throws Exception {
+        Map<String, VersionedData<Assignment>> updateAssignmentVersion = new HashMap<>();
+        for (String topoId : topoIds) {
+            Integer recordedVersion = -1;
+            Integer version = stormClusterState.assignmentVersion(topoId, callback);
+            VersionedData<Assignment> locAssignment = localAssignmentVersion.get(topoId);
+            if (locAssignment != null) {
+                recordedVersion = locAssignment.getVersion();
+            }
+            if (version == null) {
+                // ignore
+            } else if (version == recordedVersion) {
+                updateAssignmentVersion.put(topoId, locAssignment);
+            } else {
+                VersionedData<Assignment> assignmentVersion = stormClusterState.assignmentInfoWithVersion(topoId, callback);
+                updateAssignmentVersion.put(topoId, assignmentVersion);
+            }
+        }
+        return updateAssignmentVersion;
+    }
+    
+    protected Map<String, List<ProfileRequest>> getProfileActions(IStormClusterState stormClusterState, List<String> stormIds) throws Exception {
+        Map<String, List<ProfileRequest>> ret = new HashMap<String, List<ProfileRequest>>();
+        for (String stormId : stormIds) {
+            List<ProfileRequest> profileRequests = stormClusterState.getTopologyProfileRequests(stormId);
+            ret.put(stormId, profileRequests);
+        }
+        return ret;
+    }
+    
+    protected Map<Integer, LocalAssignment> readAssignments(Map<String, VersionedData<Assignment>> assignmentsSnapshot) {
+        try {
+            Map<Integer, LocalAssignment> portLA = new HashMap<>();
+            for (Map.Entry<String, VersionedData<Assignment>> assignEntry : assignmentsSnapshot.entrySet()) {
+                String topoId = assignEntry.getKey();
+                Assignment assignment = assignEntry.getValue().getData();
+
+                Map<Integer, LocalAssignment> portTasks = readMyExecutors(topoId, assignmentId, assignment);
+
+                for (Map.Entry<Integer, LocalAssignment> entry : portTasks.entrySet()) {
+
+                    Integer port = entry.getKey();
+
+                    LocalAssignment la = entry.getValue();
+
+                    if (!portLA.containsKey(port)) {
+                        portLA.put(port, la);
+                    } else {
+                        throw new RuntimeException("Should not have multiple topologies assigned to one port "
+                          + port + " " + la + " " + portLA);
+                    }
+                }
+            }
+            readRetry.set(0);
+            return portLA;
+        } catch (RuntimeException e) {
+            if (readRetry.get() > 2) {
+                throw e;
+            } else {
+                readRetry.addAndGet(1);
+            }
+            LOG.warn("{} : retrying {} of 3", e.getMessage(), readRetry.get());
+            return null;
+        }
+    }
+    
+    protected Map<Integer, LocalAssignment> readMyExecutors(String stormId, String assignmentId, Assignment assignment) {
+        Map<Integer, LocalAssignment> portTasks = new HashMap<>();
+        Map<Long, WorkerResources> slotsResources = new HashMap<>();
+        Map<NodeInfo, WorkerResources> nodeInfoWorkerResourcesMap = assignment.get_worker_resources();
+        if (nodeInfoWorkerResourcesMap != null) {
+            for (Map.Entry<NodeInfo, WorkerResources> entry : nodeInfoWorkerResourcesMap.entrySet()) {
+                if (entry.getKey().get_node().equals(assignmentId)) {
+                    Set<Long> ports = entry.getKey().get_port();
+                    for (Long port : ports) {
+                        slotsResources.put(port, entry.getValue());
+                    }
+                }
+            }
+        }
+        Map<List<Long>, NodeInfo> executorNodePort = assignment.get_executor_node_port();
+        if (executorNodePort != null) {
+            for (Map.Entry<List<Long>, NodeInfo> entry : executorNodePort.entrySet()) {
+                if (entry.getValue().get_node().equals(assignmentId)) {
+                    for (Long port : entry.getValue().get_port()) {
+                        LocalAssignment localAssignment = portTasks.get(port.intValue());
+                        if (localAssignment == null) {
+                            List<ExecutorInfo> executors = new ArrayList<>();
+                            localAssignment = new LocalAssignment(stormId, executors);
+                            if (slotsResources.containsKey(port)) {
+                                localAssignment.set_resources(slotsResources.get(port));
+                            }
+                            portTasks.put(port.intValue(), localAssignment);
+                        }
+                        List<ExecutorInfo> executorInfoList = localAssignment.get_executors();
+                        executorInfoList.add(new ExecutorInfo(entry.getKey().get(0).intValue(), entry.getKey().get(entry.getKey().size() - 1).intValue()));
+                    }
+                }
+            }
+        }
+        return portTasks;
+    }
+
+    public synchronized void shutdownAllWorkers() {
+        for (Slot slot: slots.values()) {
+            slot.setNewAssignment(null);
+        }
+
+        for (Slot slot: slots.values()) {
+            try {
+                int count = 0;
+                while (slot.getMachineState() != MachineState.EMPTY) {
+                    if (count > 10) {
+                        LOG.warn("DONE waiting for {} to finish {}", slot, slot.getMachineState());
+                        break;
+                    }
+                    if (Time.isSimulating()) {
+                        Time.advanceTime(1000);
+                        Thread.sleep(100);
+                    } else {
+                        Time.sleep(100);
+                    }
+                    count++;
+                }
+            } catch (Exception e) {
+                LOG.error("Error trying to shutdown workers in {}", slot, e);
+            }
+        }
+    }
+    
+    @Override
+    public void close() {
+        for (Slot slot: slots.values()) {
+            try {
+                slot.close();
+            } catch (Exception e) {
+                LOG.error("Error trying to shutdown {}", slot, e);
+            }
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainer.java
new file mode 100644
index 0000000..29b8576
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainer.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RunAsUserContainer extends BasicContainer {
+    private static final Logger LOG = LoggerFactory.getLogger(RunAsUserContainer.class);
+
+    public RunAsUserContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+            LocalAssignment assignment, LocalState localState,
+            String workerId) throws IOException {
+        this(type, conf, supervisorId, port, assignment, localState, workerId, null, null, null);
+    }
+    
+    RunAsUserContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+            LocalAssignment assignment, LocalState localState,
+            String workerId, Map<String, Object> topoConf, AdvancedFSOps ops, String profileCmd) throws IOException {
+        super(type, conf, supervisorId, port, assignment, localState, workerId, topoConf, ops,
+                profileCmd);
+        if (Utils.isOnWindows()) {
+            throw new UnsupportedOperationException("ERROR: Windows doesn't support running workers as different users yet");
+        }
+    }
+
+    private void signal(long pid, int signal) throws IOException {
+        List<String> commands = Arrays.asList("signal", String.valueOf(pid), String.valueOf(signal));
+        String user = getWorkerUser();
+        String logPrefix = "kill -"+signal+" " + pid;
+        SupervisorUtils.processLauncherAndWait(_conf, user, commands, null, logPrefix);
+    }
+    
+    @Override
+    protected void kill(long pid) throws IOException {
+        signal(pid, 15);
+    }
+    
+    @Override
+    protected void forceKill(long pid) throws IOException {
+        signal(pid, 9);
+    }
+    
+    @Override
+    protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix, File targetDir) throws IOException, InterruptedException {
+        String user = this.getWorkerUser();
+        String td = targetDir.getAbsolutePath();
+        LOG.info("Running as user: {} command: {}", user, command);
+        String containerFile = Utils.containerFilePath(td);
+        if (Utils.checkFileExists(containerFile)) {
+            SupervisorUtils.rmrAsUser(_conf, containerFile, containerFile);
+        }
+        String scriptFile = Utils.scriptFilePath(td);
+        if (Utils.checkFileExists(scriptFile)) {
+            SupervisorUtils.rmrAsUser(_conf, scriptFile, scriptFile);
+        }
+        String script = Utils.writeScript(td, command, env);
+        List<String> args = Arrays.asList("profiler", td, script);
+        int ret = SupervisorUtils.processLauncherAndWait(_conf, user, args, env, logPrefix);
+        return ret == 0;
+    }
+
+    @Override
+    protected void launchWorkerProcess(List<String> command, Map<String, String> env, 
+            String logPrefix, ExitCodeCallback processExitCallback, File targetDir) throws IOException {
+        String workerDir = targetDir.getAbsolutePath();
+        String user = this.getWorkerUser();
+        List<String> args = Arrays.asList("worker", workerDir, Utils.writeScript(workerDir, command, env));
+        List<String> commandPrefix = null;
+        SupervisorUtils.processLauncher(_conf, user, commandPrefix, args, null, logPrefix, processExitCallback, targetDir);
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java
new file mode 100644
index 0000000..5b446b3
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/RunAsUserContainerLauncher.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.storm.daemon.supervisor.Container.ContainerType;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.utils.LocalState;
+
+public class RunAsUserContainerLauncher extends ContainerLauncher {
+    private final Map<String, Object> _conf;
+    private final String _supervisorId;
+    
+    public RunAsUserContainerLauncher(Map<String, Object> conf, String supervisorId) throws IOException {
+        _conf = conf;
+        _supervisorId = supervisorId;
+    }
+
+    @Override
+    public Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+        Container container = new RunAsUserContainer(ContainerType.LAUNCH, _conf, _supervisorId, port, assignment,
+                state, null, null, null, null);
+        container.setup();
+        container.launch();
+        return container;
+    }
+
+    @Override
+    public Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+        return new RunAsUserContainer(ContainerType.RECOVER_FULL, _conf, _supervisorId, port, assignment,
+                state, null, null, null, null);
+    }
+    
+    @Override
+    public Killable recoverContainer(String workerId, LocalState localState) throws IOException {
+        return new RunAsUserContainer(ContainerType.RECOVER_PARTIAL, _conf, _supervisorId, -1, null,
+                localState, workerId, null, null, null);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Slot.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Slot.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Slot.java
new file mode 100644
index 0000000..c92db0c
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Slot.java
@@ -0,0 +1,785 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.storm.Config;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.localizer.ILocalizer;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Slot extends Thread implements AutoCloseable {
+    private static final Logger LOG = LoggerFactory.getLogger(Slot.class);
+    
+    static enum MachineState {
+        EMPTY,
+        RUNNING,
+        WAITING_FOR_WORKER_START,
+        KILL_AND_RELAUNCH,
+        KILL,
+        WAITING_FOR_BASIC_LOCALIZATION,
+        WAITING_FOR_BLOB_LOCALIZATION;
+    };
+    
+    static class StaticState {
+        public final ILocalizer localizer;
+        public final long hbTimeoutMs;
+        public final long firstHbTimeoutMs;
+        public final long killSleepMs;
+        public final long monitorFreqMs;
+        public final ContainerLauncher containerLauncher;
+        public final int port;
+        public final String host;
+        public final ISupervisor iSupervisor;
+        public final LocalState localState;
+        
+        StaticState(ILocalizer localizer, long hbTimeoutMs, long firstHbTimeoutMs,
+                long killSleepMs, long monitorFreqMs,
+                ContainerLauncher containerLauncher, String host, int port,
+                ISupervisor iSupervisor, LocalState localState) {
+            this.localizer = localizer;
+            this.hbTimeoutMs = hbTimeoutMs;
+            this.firstHbTimeoutMs = firstHbTimeoutMs;
+            this.containerLauncher = containerLauncher;
+            this.killSleepMs = killSleepMs;
+            this.monitorFreqMs = monitorFreqMs;
+            this.host = host;
+            this.port = port;
+            this.iSupervisor = iSupervisor;
+            this.localState = localState;
+        }
+    }
+    
+    static class DynamicState {
+        public final MachineState state;
+        public final LocalAssignment newAssignment;
+        public final LocalAssignment currentAssignment;
+        public final Container container;
+        public final LocalAssignment pendingLocalization;
+        public final Future<Void> pendingDownload;
+        public final Set<TopoProfileAction> profileActions;
+        public final Set<TopoProfileAction> pendingStopProfileActions;
+        
+        /**
+         * The last time that WAITING_FOR_WORKER_START, KILL, or KILL_AND_RELAUNCH were entered into.
+         */
+        public final long startTime;
+        
+        public DynamicState(final LocalAssignment currentAssignment, Container container, final LocalAssignment newAssignment) {
+            this.currentAssignment = currentAssignment;
+            this.container = container;
+            if ((currentAssignment == null) ^ (container == null)) {
+                throw new IllegalArgumentException("Container and current assignment must both be null, or neither can be null");
+            }
+            
+            if (currentAssignment == null) {
+                state = MachineState.EMPTY;
+            } else {
+                state = MachineState.RUNNING;
+            }
+            
+            this.startTime = System.currentTimeMillis();
+            this.newAssignment = newAssignment;
+            this.pendingLocalization = null;
+            this.pendingDownload = null;
+            this.profileActions = new HashSet<>();
+            this.pendingStopProfileActions = new HashSet<>();
+        }
+        
+        public DynamicState(final MachineState state, final LocalAssignment newAssignment,
+                final Container container, final LocalAssignment currentAssignment,
+                final LocalAssignment pendingLocalization, final long startTime,
+                final Future<Void> pendingDownload, final Set<TopoProfileAction> profileActions, 
+                final Set<TopoProfileAction> pendingStopProfileActions) {
+            this.state = state;
+            this.newAssignment = newAssignment;
+            this.currentAssignment = currentAssignment;
+            this.container = container;
+            this.pendingLocalization = pendingLocalization;
+            this.startTime = startTime;
+            this.pendingDownload = pendingDownload;
+            this.profileActions = profileActions;
+            this.pendingStopProfileActions = pendingStopProfileActions;
+        }
+        
+        public String toString() {
+            StringBuffer sb = new StringBuffer();
+            sb.append(state);
+            sb.append(" msInState: ");
+            sb.append(Time.currentTimeMillis() - startTime);
+            if (container != null) {
+                sb.append(" ");
+                sb.append(container);
+            }
+            return sb.toString();
+        }
+
+        /**
+         * Set the new assignment for the state.  This should never be called from within the state machine.
+         * It is an input from outside.
+         * @param newAssignment the new assignment to set
+         * @return the updated DynamicState.
+         */
+        public DynamicState withNewAssignment(LocalAssignment newAssignment) {
+            return new DynamicState(this.state, newAssignment,
+                    this.container, this.currentAssignment,
+                    this.pendingLocalization, this.startTime,
+                    this.pendingDownload, this.profileActions,
+                    this.pendingStopProfileActions);
+        }
+        
+        public DynamicState withPendingLocalization(LocalAssignment pendingLocalization, Future<Void> pendingDownload) {
+            return new DynamicState(this.state, this.newAssignment,
+                    this.container, this.currentAssignment,
+                    pendingLocalization, this.startTime,
+                    pendingDownload, this.profileActions,
+                    this.pendingStopProfileActions);
+        }
+        
+        public DynamicState withPendingLocalization(Future<Void> pendingDownload) {
+            return withPendingLocalization(this.pendingLocalization, pendingDownload);
+        }
+        
+        public DynamicState withState(final MachineState state) {
+            long newStartTime = Time.currentTimeMillis();
+            return new DynamicState(state, this.newAssignment,
+                    this.container, this.currentAssignment,
+                    this.pendingLocalization, newStartTime,
+                    this.pendingDownload, this.profileActions,
+                    this.pendingStopProfileActions);
+        }
+
+        public DynamicState withCurrentAssignment(final Container container, final LocalAssignment currentAssignment) {
+            return new DynamicState(this.state, this.newAssignment,
+                    container, currentAssignment,
+                    this.pendingLocalization, this.startTime,
+                    this.pendingDownload, this.profileActions,
+                    this.pendingStopProfileActions);
+        }
+
+        public DynamicState withProfileActions(Set<TopoProfileAction> profileActions, Set<TopoProfileAction> pendingStopProfileActions) {
+            return new DynamicState(this.state, this.newAssignment,
+                    this.container, this.currentAssignment,
+                    this.pendingLocalization, this.startTime,
+                    this.pendingDownload, profileActions,
+                    pendingStopProfileActions);
+        }
+    };
+    
+    static class TopoProfileAction {
+        public final String topoId;
+        public final ProfileRequest request;
+        
+        public TopoProfileAction(String topoId, ProfileRequest request) {
+            this.topoId = topoId;
+            this.request = request;
+        }
+        
+        @Override
+        public int hashCode() {
+            return (37 * topoId.hashCode()) + request.hashCode(); 
+        }
+        
+        @Override
+        public boolean equals(Object other) {
+            if (!(other instanceof TopoProfileAction)) {
+                return false;
+            }
+            TopoProfileAction o = (TopoProfileAction) other;
+            return topoId.equals(o.topoId) && request.equals(o.request);
+        }
+        
+        @Override
+        public String toString() {
+            return "{ " + topoId + ": " + request + " }";
+        }
+    }
+    
+    static boolean equivalent(LocalAssignment a, LocalAssignment b) {
+        if (a == null && b == null) {
+            return true;
+        }
+        if (a != null && b != null) {
+            if (a.get_topology_id().equals(b.get_topology_id())) {
+                Set<ExecutorInfo> aexec = new HashSet<>(a.get_executors());
+                Set<ExecutorInfo> bexec = new HashSet<>(b.get_executors());
+                if (aexec.equals(bexec)) {
+                    boolean aHasResources = a.is_set_resources();
+                    boolean bHasResources = b.is_set_resources();
+                    if (!aHasResources && !bHasResources) {
+                        return true;
+                    }
+                    if (aHasResources && bHasResources) {
+                        if (a.get_resources().equals(b.get_resources())) {
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+        return false;
+    }
+    
+    static DynamicState stateMachineStep(DynamicState dynamicState, StaticState staticState) throws Exception {
+        LOG.debug("STATE {}", dynamicState.state);
+        switch (dynamicState.state) {
+            case EMPTY:
+                return handleEmpty(dynamicState, staticState);
+            case RUNNING:
+                return handleRunning(dynamicState, staticState);
+            case WAITING_FOR_WORKER_START:
+                return handleWaitingForWorkerStart(dynamicState, staticState);
+            case KILL_AND_RELAUNCH:
+                return handleKillAndRelaunch(dynamicState, staticState);
+            case KILL:
+                return handleKill(dynamicState, staticState);
+            case WAITING_FOR_BASIC_LOCALIZATION:
+                return handleWaitingForBasicLocalization(dynamicState, staticState);
+            case WAITING_FOR_BLOB_LOCALIZATION:
+                return handleWaitingForBlobLocalization(dynamicState, staticState);
+            default:
+                throw new IllegalStateException("Code not ready to handle a state of "+dynamicState.state);
+        }
+    }
+    
+    /**
+     * Prepare for a new assignment by downloading new required blobs, or going to empty if there is nothing to download.
+     * PRECONDITION: The slot should be empty
+     * @param dynamicState current state
+     * @param staticState static data
+     * @return the next state
+     * @throws IOException on any error
+     */
+    static DynamicState prepareForNewAssignmentNoWorkersRunning(DynamicState dynamicState, StaticState staticState) throws IOException {
+        assert(dynamicState.container == null);
+        
+        if (dynamicState.newAssignment == null) {
+            return dynamicState.withState(MachineState.EMPTY);
+        }
+        Future<Void> pendingDownload = staticState.localizer.requestDownloadBaseTopologyBlobs(dynamicState.newAssignment, staticState.port);
+        return dynamicState.withPendingLocalization(dynamicState.newAssignment, pendingDownload).withState(MachineState.WAITING_FOR_BASIC_LOCALIZATION);
+    }
+    
+    /**
+     * Kill the current container and start downloading what the new assignment needs, if there is a new assignment
+     * PRECONDITION: container != null
+     * @param dynamicState current state
+     * @param staticState static data
+     * @return the next state
+     * @throws Exception 
+     */
+    static DynamicState killContainerForChangedAssignment(DynamicState dynamicState, StaticState staticState) throws Exception {
+        assert(dynamicState.container != null);
+        
+        staticState.iSupervisor.killedWorker(staticState.port);
+        dynamicState.container.kill();
+        Future<Void> pendingDownload = null;
+        if (dynamicState.newAssignment != null) {
+            pendingDownload = staticState.localizer.requestDownloadBaseTopologyBlobs(dynamicState.newAssignment, staticState.port);
+        }
+        Time.sleep(staticState.killSleepMs);
+        return dynamicState.withPendingLocalization(dynamicState.newAssignment, pendingDownload).withState(MachineState.KILL);
+    }
+    
+    /**
+     * Kill the current container and relaunch it.  (Something odd happened)
+     * PRECONDITION: container != null
+     * @param dynamicState current state
+     * @param staticState static data
+     * @return the next state
+     * @throws Exception 
+     */
+    static DynamicState killAndRelaunchContainer(DynamicState dynamicState, StaticState staticState) throws Exception {
+        assert(dynamicState.container != null);
+        
+        dynamicState.container.kill();
+        Time.sleep(staticState.killSleepMs);
+        
+        //any stop profile actions that hadn't timed out yet, we should restart after the worker is running again.
+        HashSet<TopoProfileAction> mod = new HashSet<>(dynamicState.profileActions);
+        mod.addAll(dynamicState.pendingStopProfileActions);
+        return dynamicState.withState(MachineState.KILL_AND_RELAUNCH).withProfileActions(mod, Collections.<TopoProfileAction> emptySet());
+    }
+    
+    /**
+     * Clean up a container
+     * PRECONDITION: All of the processes have died.
+     * @param dynamicState current state
+     * @param staticState static data
+     * @param nextState the next MachineState to go to.
+     * @return the next state.
+     */
+    static DynamicState cleanupCurrentContainer(DynamicState dynamicState, StaticState staticState, MachineState nextState) throws Exception {
+        assert(dynamicState.container != null);
+        assert(dynamicState.currentAssignment != null);
+        assert(dynamicState.container.areAllProcessesDead());
+        
+        dynamicState.container.cleanUp();
+        staticState.localizer.releaseSlotFor(dynamicState.currentAssignment, staticState.port);
+        DynamicState ret = dynamicState.withCurrentAssignment(null, null);
+        if (nextState != null) {
+            ret = ret.withState(nextState);
+        }
+        return ret;
+    }
+    
+    /**
+     * State Transitions for WAITING_FOR_BLOB_LOCALIZATION state.
+     * PRECONDITION: neither pendingLocalization nor pendingDownload is null.
+     * PRECONDITION: The slot should be empty
+     * @param dynamicState current state
+     * @param staticState static data
+     * @return the next state
+     * @throws Exception on any error
+     */
+    static DynamicState handleWaitingForBlobLocalization(DynamicState dynamicState, StaticState staticState) throws Exception {
+        assert(dynamicState.pendingLocalization != null);
+        assert(dynamicState.pendingDownload != null);
+        assert(dynamicState.container == null);
+        
+        //Ignore changes to scheduling while downloading the topology blobs
+        // We don't support canceling the download through the future yet,
+        // so to keep everything in sync, just wait
+        try {
+            dynamicState.pendingDownload.get(1000, TimeUnit.MILLISECONDS);
+            //Downloading of all blobs finished.
+            if (!equivalent(dynamicState.newAssignment, dynamicState.pendingLocalization)) {
+                //Scheduling changed
+                staticState.localizer.releaseSlotFor(dynamicState.pendingLocalization, staticState.port);
+                return prepareForNewAssignmentNoWorkersRunning(dynamicState, staticState);
+            }
+            Container c = staticState.containerLauncher.launchContainer(staticState.port, dynamicState.pendingLocalization, staticState.localState);
+            return dynamicState.withCurrentAssignment(c, dynamicState.pendingLocalization).withState(MachineState.WAITING_FOR_WORKER_START).withPendingLocalization(null, null);
+        } catch (TimeoutException e) {
+            //We waited for 1 second loop around and try again....
+            return dynamicState;
+        }
+    }
+    
+    /**
+     * State Transitions for WAITING_FOR_BASIC_LOCALIZATION state.
+     * PRECONDITION: neither pendingLocalization nor pendingDownload is null.
+     * PRECONDITION: The slot should be empty
+     * @param dynamicState current state
+     * @param staticState static data
+     * @return the next state
+     * @throws Exception on any error
+     */
+    static DynamicState handleWaitingForBasicLocalization(DynamicState dynamicState, StaticState staticState) throws Exception {
+        assert(dynamicState.pendingLocalization != null);
+        assert(dynamicState.pendingDownload != null);
+        assert(dynamicState.container == null);
+        
+        //Ignore changes to scheduling while downloading the topology code
+        // We don't support canceling the download through the future yet,
+        // so to keep everything in sync, just wait
+        try {
+            dynamicState.pendingDownload.get(1000, TimeUnit.MILLISECONDS);
+            Future<Void> pendingDownload = staticState.localizer.requestDownloadTopologyBlobs(dynamicState.pendingLocalization, staticState.port);
+            return dynamicState.withPendingLocalization(pendingDownload).withState(MachineState.WAITING_FOR_BLOB_LOCALIZATION);
+        } catch (TimeoutException e) {
+            return dynamicState;
+        }
+    }
+
+    /**
+     * State Transitions for KILL state.
+     * PRECONDITION: container.kill() was called
+     * PRECONDITION: container != null && currentAssignment != null
+     * @param dynamicState current state
+     * @param staticState static data
+     * @return the next state
+     * @throws Exception on any error
+     */
+    static DynamicState handleKill(DynamicState dynamicState, StaticState staticState) throws Exception {
+        assert(dynamicState.container != null);
+        assert(dynamicState.currentAssignment != null);
+        
+        if (dynamicState.container.areAllProcessesDead()) {
+            LOG.warn("SLOT {} all processes are dead...", staticState.port);
+            return cleanupCurrentContainer(dynamicState, staticState, 
+                    dynamicState.pendingLocalization == null ? MachineState.EMPTY : MachineState.WAITING_FOR_BASIC_LOCALIZATION);
+        }
+
+        LOG.warn("SLOT {} force kill and wait...", staticState.port);
+        dynamicState.container.forceKill();
+        Time.sleep(staticState.killSleepMs);
+        return dynamicState;
+    }
+
+    /**
+     * State Transitions for KILL_AND_RELAUNCH state.
+     * PRECONDITION: container.kill() was called
+     * PRECONDITION: container != null && currentAssignment != null
+     * @param dynamicState current state
+     * @param staticState static data
+     * @return the next state
+     * @throws Exception on any error
+     */
+    static DynamicState handleKillAndRelaunch(DynamicState dynamicState, StaticState staticState) throws Exception {
+        assert(dynamicState.container != null);
+        assert(dynamicState.currentAssignment != null);
+        
+        if (dynamicState.container.areAllProcessesDead()) {
+            if (equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
+                dynamicState.container.cleanUpForRestart();
+                dynamicState.container.relaunch();
+                return dynamicState.withState(MachineState.WAITING_FOR_WORKER_START);
+            }
+            //Scheduling changed after we killed all of the processes
+            return prepareForNewAssignmentNoWorkersRunning(cleanupCurrentContainer(dynamicState, staticState, null), staticState);
+        }
+        //The child processes typically exit in < 1 sec.  If 2 mins later they are still around something is wrong
+        if ((Time.currentTimeMillis() - dynamicState.startTime) > 120_000) {
+            throw new RuntimeException("Not all processes in " + dynamicState.container + " exited after 120 seconds");
+        }
+        dynamicState.container.forceKill();
+        Time.sleep(staticState.killSleepMs);
+        return dynamicState;
+    }
+
+    /**
+     * State Transitions for WAITING_FOR_WORKER_START state.
+     * PRECONDITION: container != null && currentAssignment != null
+     * @param dynamicState current state
+     * @param staticState static data
+     * @return the next state
+     * @throws Exception on any error
+     */
+    static DynamicState handleWaitingForWorkerStart(DynamicState dynamicState, StaticState staticState) throws Exception {
+        assert(dynamicState.container != null);
+        assert(dynamicState.currentAssignment != null);
+        
+        LSWorkerHeartbeat hb = dynamicState.container.readHeartbeat();
+        if (hb != null) {
+            long hbAgeMs = (Time.currentTimeSecs() - hb.get_time_secs()) * 1000;
+            if (hbAgeMs <= staticState.hbTimeoutMs) {
+                return dynamicState.withState(MachineState.RUNNING);
+            }
+        }
+        
+        if (!equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
+            //We were rescheduled while waiting for the worker to come up
+            LOG.warn("SLOT {}: Assignment Changed from {} to {}", staticState.port, dynamicState.currentAssignment, dynamicState.newAssignment);
+            return killContainerForChangedAssignment(dynamicState, staticState);
+        }
+        
+        long timeDiffms = (Time.currentTimeMillis() - dynamicState.startTime);
+        if (timeDiffms > staticState.firstHbTimeoutMs) {
+            LOG.warn("SLOT {}: Container {} failed to launch in {} ms.", staticState.port, dynamicState.container, staticState.firstHbTimeoutMs);
+            return killAndRelaunchContainer(dynamicState, staticState);
+        }
+        Time.sleep(1000);
+        return dynamicState;
+    }
+
+    /**
+     * State Transitions for RUNNING state.
+     * PRECONDITION: container != null && currentAssignment != null
+     * @param dynamicState current state
+     * @param staticState static data
+     * @return the next state
+     * @throws Exception on any error
+     */
+    static DynamicState handleRunning(DynamicState dynamicState, StaticState staticState) throws Exception {
+        assert(dynamicState.container != null);
+        assert(dynamicState.currentAssignment != null);
+        
+        if (!equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
+            LOG.warn("SLOT {}: Assignment Changed from {} to {}", staticState.port, dynamicState.currentAssignment, dynamicState.newAssignment);
+            //Scheduling changed while running...
+            return killContainerForChangedAssignment(dynamicState, staticState);
+        }
+        if (dynamicState.container.didMainProcessExit()) {
+            LOG.warn("SLOT {}: main process has exited", staticState.port);
+            return killAndRelaunchContainer(dynamicState, staticState);
+        }
+        
+        LSWorkerHeartbeat hb = dynamicState.container.readHeartbeat();
+        if (hb == null) {
+            LOG.warn("SLOT {}: HB returned as null", staticState.port);
+            //This can happen if the supervisor crashed after launching a
+            // worker that never came up.
+            return killAndRelaunchContainer(dynamicState, staticState);
+        }
+        
+        long timeDiffMs = (Time.currentTimeSecs() - hb.get_time_secs()) * 1000;
+        if (timeDiffMs > staticState.hbTimeoutMs) {
+            LOG.warn("SLOT {}: HB is too old {} > {}", staticState.port, timeDiffMs, staticState.hbTimeoutMs);
+            return killAndRelaunchContainer(dynamicState, staticState);
+        }
+        
+        //The worker is up and running check for profiling requests
+        if (!dynamicState.profileActions.isEmpty()) {
+            HashSet<TopoProfileAction> mod = new HashSet<>(dynamicState.profileActions);
+            HashSet<TopoProfileAction> modPending = new HashSet<>(dynamicState.pendingStopProfileActions);
+            Iterator<TopoProfileAction> iter = mod.iterator();
+            while (iter.hasNext()) {
+                TopoProfileAction action = iter.next();
+                if (!action.topoId.equals(dynamicState.currentAssignment.get_topology_id())) {
+                    iter.remove();
+                    LOG.warn("Dropping {} wrong topology is running", action);
+                    //Not for this topology so skip it
+                } else {
+                    if (modPending.contains(action)) {
+                        boolean isTimeForStop = Time.currentTimeMillis() > action.request.get_time_stamp();
+                        if (isTimeForStop) {
+                            if (dynamicState.container.runProfiling(action.request, true)) {
+                                LOG.debug("Stopped {} action finished", action);
+                                iter.remove();
+                                modPending.remove(action);
+                            } else {
+                                LOG.warn("Stopping {} failed, will be retried", action);
+                            }
+                        } else {
+                            LOG.debug("Still pending {} now: {}", action, Time.currentTimeMillis());
+                        }
+                    } else {
+                        //J_PROFILE_START is not used.  When you see a J_PROFILE_STOP
+                        // start profiling and save it away to stop when timeout happens
+                        if (action.request.get_action() == ProfileAction.JPROFILE_STOP) {
+                            if (dynamicState.container.runProfiling(action.request, false)) {
+                                modPending.add(action);
+                                LOG.debug("Started {} now: {}", action, Time.currentTimeMillis());
+                            } else {
+                                LOG.warn("Starting {} failed, will be retried", action);
+                            }
+                        } else {
+                            if (dynamicState.container.runProfiling(action.request, false)) {
+                                LOG.debug("Started {} action finished", action);
+                                iter.remove();
+                            } else {
+                                LOG.warn("Starting {} failed, will be retried", action);
+                            }
+                        }
+                    }
+                }
+            }
+            dynamicState = dynamicState.withProfileActions(mod, modPending);
+        }
+        Time.sleep(staticState.monitorFreqMs);
+        return dynamicState;
+    }
+
+    static DynamicState handleEmpty(DynamicState dynamicState, StaticState staticState) throws InterruptedException, IOException {
+        if (!equivalent(dynamicState.newAssignment, dynamicState.currentAssignment)) {
+            return prepareForNewAssignmentNoWorkersRunning(dynamicState, staticState);
+        }
+        //Both assignments are null, just wait
+        if (dynamicState.profileActions != null && !dynamicState.profileActions.isEmpty()) {
+            //Nothing is scheduled here so throw away all of the profileActions
+            LOG.warn("Dropping {} no topology is running", dynamicState.profileActions);
+            dynamicState = dynamicState.withProfileActions(Collections.<TopoProfileAction> emptySet(), Collections.<TopoProfileAction> emptySet());
+        }
+        Time.sleep(1000);
+        return dynamicState;
+    }
+    
+    private final AtomicReference<LocalAssignment> newAssignment = new AtomicReference<>();
+    private final AtomicReference<Set<TopoProfileAction>> profiling =
+            new AtomicReference<Set<TopoProfileAction>>(new HashSet<TopoProfileAction>());
+    private final StaticState staticState;
+    private final IStormClusterState clusterState;
+    private volatile boolean done = false;
+    private volatile DynamicState dynamicState;
+    private final AtomicReference<Map<Long, LocalAssignment>> cachedCurrentAssignments;
+    
+    public Slot(ILocalizer localizer, Map<String, Object> conf, 
+            ContainerLauncher containerLauncher, String host,
+            int port, LocalState localState,
+            IStormClusterState clusterState,
+            ISupervisor iSupervisor,
+            AtomicReference<Map<Long, LocalAssignment>> cachedCurrentAssignments) throws Exception {
+        super("SLOT_"+port);
+
+        this.cachedCurrentAssignments = cachedCurrentAssignments;
+        this.clusterState = clusterState;
+        Map<Integer, LocalAssignment> assignments = localState.getLocalAssignmentsMap();
+        LocalAssignment currentAssignment = null;
+        if (assignments != null) {
+            currentAssignment = assignments.get(port);
+        }
+        Container container = null;
+        if (currentAssignment != null) { 
+            try {
+                container = containerLauncher.recoverContainer(port, currentAssignment, localState);
+            } catch (ContainerRecoveryException e) {
+                //We could not recover container will be null.
+            }
+        }
+        
+        LocalAssignment newAssignment = currentAssignment;
+        if (currentAssignment != null && container == null) {
+            currentAssignment = null;
+            //Assigned something but it is not running
+        }
+        
+        dynamicState = new DynamicState(currentAssignment, container, newAssignment);
+        staticState = new StaticState(localizer, 
+                Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS)) * 1000,
+                Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_START_TIMEOUT_SECS)) * 1000,
+                Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS)) * 1000,
+                Utils.getInt(conf.get(Config.SUPERVISOR_MONITOR_FREQUENCY_SECS)) * 1000,
+                containerLauncher,
+                host,
+                port,
+                iSupervisor,
+                localState);
+        this.newAssignment.set(dynamicState.newAssignment);
+        if (MachineState.RUNNING == dynamicState.state) {
+            //We are running so we should recover the blobs.
+            staticState.localizer.recoverRunningTopology(currentAssignment, port);
+            saveNewAssignment(currentAssignment);
+        }
+        LOG.warn("SLOT {}:{} Starting in state {} - assignment {}", staticState.host, staticState.port, dynamicState.state, dynamicState.currentAssignment);
+    }
+    
+    public MachineState getMachineState() {
+        return dynamicState.state;
+    }
+    
+    /**
+     * Set a new assignment asynchronously
+     * @param newAssignment the new assignment for this slot to run, null to run nothing
+     */
+    public void setNewAssignment(LocalAssignment newAssignment) {
+        this.newAssignment.set(newAssignment);
+    }
+    
+    public void addProfilerActions(Set<TopoProfileAction> actions) {
+        if (actions != null) {
+            while(true) {
+                Set<TopoProfileAction> orig = profiling.get();
+                Set<TopoProfileAction> newActions = new HashSet<>(orig);
+                newActions.addAll(actions);
+                if (profiling.compareAndSet(orig, newActions)) {
+                    return;
+                }
+            }
+        }
+    }
+    
+    public String getWorkerId() {
+        String workerId = null;
+        Container c = dynamicState.container;
+        if (c != null) {
+            workerId = c.getWorkerId();
+        }
+        return workerId;
+    }
+    
+    private void saveNewAssignment(LocalAssignment assignment) {
+        synchronized(staticState.localState) {
+            Map<Integer, LocalAssignment> assignments = staticState.localState.getLocalAssignmentsMap();
+            if (assignments == null) {
+                assignments = new HashMap<>();
+            }
+            if (assignment == null) {
+                assignments.remove(staticState.port);
+            } else {
+                assignments.put(staticState.port, assignment);
+            }
+            staticState.localState.setLocalAssignmentsMap(assignments);
+        }
+        Map<Long, LocalAssignment> update = null;
+        Map<Long, LocalAssignment> orig = null;
+        do {
+            Long lport = new Long(staticState.port);
+            orig = cachedCurrentAssignments.get();
+            update = new HashMap<>(orig);
+            if (assignment == null) {
+                update.remove(lport);
+            } else {
+                update.put(lport, assignment);
+            }
+        } while (!cachedCurrentAssignments.compareAndSet(orig, update));
+    }
+    
+    public void run() {
+        try {
+            while(!done) {
+                Set<TopoProfileAction> origProfileActions = new HashSet<>(profiling.get());
+                Set<TopoProfileAction> removed = new HashSet<>(origProfileActions);
+                
+                DynamicState nextState = 
+                        stateMachineStep(dynamicState.withNewAssignment(newAssignment.get())
+                                .withProfileActions(origProfileActions, dynamicState.pendingStopProfileActions), staticState);
+
+                if (LOG.isDebugEnabled() || dynamicState.state != nextState.state) {
+                    LOG.info("STATE {} -> {}", dynamicState, nextState);
+                }
+                //Save the current state for recovery
+                if (!equivalent(nextState.currentAssignment, dynamicState.currentAssignment)) {
+                    LOG.info("SLOT {}: Changing current assignment from {} to {}", staticState.port, dynamicState.currentAssignment, nextState.currentAssignment);
+                    saveNewAssignment(nextState.currentAssignment);
+                }
+                
+                // clean up the profiler actions that are not being processed
+                removed.removeAll(dynamicState.profileActions);
+                removed.removeAll(dynamicState.pendingStopProfileActions);
+                for (TopoProfileAction action: removed) {
+                    try {
+                        clusterState.deleteTopologyProfileRequests(action.topoId, action.request);
+                    } catch (Exception e) {
+                        LOG.error("Error trying to remove profiling request, it will be retried", e);
+                    }
+                }
+                Set<TopoProfileAction> orig, copy;
+                do {
+                    orig = profiling.get();
+                    copy = new HashSet<>(orig);
+                    copy.removeAll(removed);
+                } while (!profiling.compareAndSet(orig, copy));
+                dynamicState = nextState;
+            }
+        } catch (Throwable e) {
+            if (!Utils.exceptionCauseIsInstanceOf(InterruptedException.class, e)) {
+                LOG.error("Error when processing event", e);
+                Utils.exitProcess(20, "Error when processing an event");
+            }
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        done = true;
+        this.interrupt();
+        this.join();
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java
new file mode 100644
index 0000000..c14b060
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/StandaloneSupervisor.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.storm.Config;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+public class StandaloneSupervisor implements ISupervisor {
+    private String supervisorId;
+    private Map conf;
+
+    @Override
+    public void prepare(Map stormConf, String schedulerLocalDir) {
+        try {
+            LocalState localState = new LocalState(schedulerLocalDir);
+            String supervisorId = localState.getSupervisorId();
+            if (supervisorId == null) {
+                supervisorId = generateSupervisorId();
+                localState.setSupervisorId(supervisorId);
+            }
+            this.conf = stormConf;
+            this.supervisorId = supervisorId;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public String getSupervisorId() {
+        return supervisorId;
+    }
+
+    @Override
+    public String getAssignmentId() {
+        return supervisorId;
+    }
+
+    @Override
+    public Object getMetadata() {
+        Object ports = conf.get(Config.SUPERVISOR_SLOTS_PORTS);
+        return ports;
+    }
+
+    @Override
+    public boolean confirmAssigned(int port) {
+        return true;
+    }
+
+    @Override
+    public void killedWorker(int port) {
+
+    }
+
+    @Override
+    public void assigned(Collection<Integer> ports) {
+
+    }
+
+    public String generateSupervisorId(){
+        return Utils.uuid();
+    }
+}


[19/21] storm git commit: Removed healthcheck.clj. Is not needed

Posted by ka...@apache.org.
Removed healthcheck.clj.  Is not needed


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/df33efd6
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/df33efd6
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/df33efd6

Branch: refs/heads/1.x-branch
Commit: df33efd698318d70c4c798cc31748828992aae50
Parents: 81cd074
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Oct 26 15:43:37 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Oct 26 15:43:37 2016 -0500

----------------------------------------------------------------------
 .../org/apache/storm/command/healthcheck.clj    | 88 --------------------
 1 file changed, 88 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/df33efd6/storm-core/src/clj/org/apache/storm/command/healthcheck.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/command/healthcheck.clj b/storm-core/src/clj/org/apache/storm/command/healthcheck.clj
deleted file mode 100644
index d96d7b3..0000000
--- a/storm-core/src/clj/org/apache/storm/command/healthcheck.clj
+++ /dev/null
@@ -1,88 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements.  See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership.  The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License.  You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.command.healthcheck
-  (:require [org.apache.storm
-             [config :refer :all]
-             [log :refer :all]]
-            [clojure.java [io :as io]]
-            [clojure [string :refer [split]]])
-  (:gen-class))
-
-(defn interrupter
-  "Interrupt a given thread after ms milliseconds."
-  [thread ms]
-  (let [interrupter (Thread.
-                     (fn []
-                       (try
-                         (Thread/sleep ms)
-                         (.interrupt thread)
-                         (catch InterruptedException e))))]
-    (.start interrupter)
-    interrupter))
-
-(defn check-output [lines]
-  (if (some #(.startsWith % "ERROR") lines)
-    :failed
-    :success))
-
-(defn process-script [conf script]
-  (let [script-proc (. (Runtime/getRuntime) (exec script))
-        curthread (Thread/currentThread)
-        interrupter-thread (interrupter curthread
-                                        (conf STORM-HEALTH-CHECK-TIMEOUT-MS))]
-    (try
-      (.waitFor script-proc)
-      (.interrupt interrupter-thread)
-      (if (not (= (.exitValue script-proc) 0))
-        :failed_with_exit_code
-        (check-output (split
-                       (slurp (.getInputStream script-proc))
-                       #"\n+")))
-      (catch InterruptedException e
-        (println "Script" script "timed out.")
-        :timeout)
-      (catch Exception e
-        (println "Script failed with exception: " e)
-        :failed_with_exception)
-      (finally (.interrupt interrupter-thread)))))
-
-(defn health-check [conf]
-  (let [health-dir (absolute-healthcheck-dir conf)
-        health-files (file-seq (io/file health-dir))
-        health-scripts (filter #(and (.canExecute %)
-                                     (not (.isDirectory %)))
-                               health-files)
-        results (->> health-scripts
-                     (map #(.getAbsolutePath %))
-                     (map (partial process-script conf)))]
-    (log-message
-     (pr-str (map #'vector
-                  (map #(.getAbsolutePath %) health-scripts)
-                  results)))
-    ; failed_with_exit_code is OK. We're mimicing Hadoop's health checks.
-    ; We treat non-zero exit codes as indicators that the scripts failed
-    ; to execute properly, not that the system is unhealthy, in which case
-    ; we don't want to start killing things.
-    (if (every? #(or (= % :failed_with_exit_code)
-                     (= % :success))
-                results)
-      0
-      1)))
-
-(defn -main [& args]
-  (let [conf (read-storm-config)]
-    (System/exit
-     (health-check conf))))


[18/21] storm git commit: Merge branch 'STORM-2131' of https://github.com/knusbaum/incubator-storm

Posted by ka...@apache.org.
Merge branch 'STORM-2131' of https://github.com/knusbaum/incubator-storm


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/81cd0745
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/81cd0745
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/81cd0745

Branch: refs/heads/1.x-branch
Commit: 81cd0745260ca066d9cf5e73ac4a34fa9ef60d55
Parents: 1d4d414
Author: Kyle Nusbaum <Ky...@gmail.com>
Authored: Tue Oct 11 11:01:31 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Oct 25 14:10:35 2016 -0500

----------------------------------------------------------------------
 .../storm/daemon/supervisor/AdvancedFSOps.java  | 15 +++++++
 .../storm/daemon/supervisor/Container.java      |  2 +-
 .../daemon/supervisor/SupervisorUtils.java      | 12 ++++-
 .../src/native/worker-launcher/impl/main.c      | 20 ++++++++-
 .../worker-launcher/impl/worker-launcher.c      | 46 ++++++++++++--------
 .../worker-launcher/impl/worker-launcher.h      |  2 +-
 6 files changed, 75 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
index 361328e..e531b84 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
@@ -95,6 +95,11 @@ public class AdvancedFSOps {
         public void setupStormCodeDir(Map<String, Object> topologyConf, File path) throws IOException {
             SupervisorUtils.setupStormCodeDir(_conf, topologyConf, path.getCanonicalPath());
         }
+
+        @Override
+        public void setupWorkerArtifactsDir(Map<String, Object> topologyConf, File path) throws IOException {
+            SupervisorUtils.setupWorkerArtifactsDir(_conf, topologyConf, path.getCanonicalPath());
+        }
     }
     
     /**
@@ -225,6 +230,16 @@ public class AdvancedFSOps {
     }
 
     /**
+     * Setup the permissions for the worker artifacts dirs
+     * @param topologyConf the config of the Topology
+     * @param path the directory to set the permissions on
+     * @throws IOException on any error
+     */
+    public void setupWorkerArtifactsDir(Map<String, Object> topologyConf, File path) throws IOException {
+        //By default this is a NOOP
+    }
+
+    /**
      * Sanity check if everything the topology needs is there for it to run.
      * @param conf the config of the supervisor
      * @param topologyId the ID of the topology

http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
index aea4b53..f4eb5e0 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
@@ -312,7 +312,7 @@ public abstract class Container implements Killable {
         File workerArtifacts = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port));
         if (!_ops.fileExists(workerArtifacts)) {
             _ops.forceMkdir(workerArtifacts);
-            _ops.setupStormCodeDir(_topoConf, workerArtifacts);
+            _ops.setupWorkerArtifactsDir(_topoConf, workerArtifacts);
         }
     
         String user = getWorkerUser();

http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
index 0784631..19d3b78 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
@@ -96,7 +96,7 @@ public class SupervisorUtils {
 
     public static void setupStormCodeDir(Map<String, Object> conf, Map<String, Object> stormConf, String dir) throws IOException {
         if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
-            String logPrefix = "setup conf for " + dir;
+            String logPrefix = "Storm Code Dir Setup for " + dir;
             List<String> commands = new ArrayList<>();
             commands.add("code-dir");
             commands.add(dir);
@@ -104,6 +104,16 @@ public class SupervisorUtils {
         }
     }
 
+    public static void setupWorkerArtifactsDir(Map<String, Object> conf, Map<String, Object> stormConf, String dir) throws IOException {
+        if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
+            String logPrefix = "Worker Artifacts Setup for " + dir;
+            List<String> commands = new ArrayList<>();
+            commands.add("artifacts-dir");
+            commands.add(dir);
+            processLauncherAndWait(conf, (String) (stormConf.get(Config.TOPOLOGY_SUBMITTER_USER)), commands, null, logPrefix);
+        }
+    }
+
     public static void rmrAsUser(Map<String, Object> conf, String id, String path) throws IOException {
         String user = Utils.getFileOwner(path);
         String logPreFix = "rmr " + id;

http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/native/worker-launcher/impl/main.c
----------------------------------------------------------------------
diff --git a/storm-core/src/native/worker-launcher/impl/main.c b/storm-core/src/native/worker-launcher/impl/main.c
index a51f9f9..b14db45 100644
--- a/storm-core/src/native/worker-launcher/impl/main.c
+++ b/storm-core/src/native/worker-launcher/impl/main.c
@@ -155,7 +155,23 @@ int main(int argc, char **argv) {
       fflush(ERRORFILE);
       return INVALID_ARGUMENT_NUMBER;
     }
-    exit_code = setup_stormdist_dir(argv[optind]);
+    exit_code = setup_dir_permissions(argv[optind], 0);
+  } else if (strcasecmp("artifacts-dir", command) == 0) {
+    if (argc != 4) {
+      fprintf(ERRORFILE, "Incorrect number of arguments (%d vs 4) for artifacts-dir\n",
+	      argc);
+      fflush(ERRORFILE);
+      return INVALID_ARGUMENT_NUMBER;
+    }
+    exit_code = setup_dir_permissions(argv[optind], 1);
+  } else if (strcasecmp("blob", command) == 0) {
+      if (argc != 4) {
+          fprintf(ERRORFILE, "Incorrect number of arguments (%d vs 4) for blob\n",
+                  argc);
+          fflush(ERRORFILE);
+          return INVALID_ARGUMENT_NUMBER;
+      }
+      exit_code = setup_dir_permissions(argv[optind], 0);
   } else if (strcasecmp("rmr", command) == 0) {
     if (argc != 4) {
       fprintf(ERRORFILE, "Incorrect number of arguments (%d vs 4) for rmr\n",
@@ -173,7 +189,7 @@ int main(int argc, char **argv) {
       return INVALID_ARGUMENT_NUMBER;
     }
     working_dir = argv[optind++];
-    exit_code = setup_stormdist_dir(working_dir);
+    exit_code = setup_dir_permissions(working_dir, 1);
     if (exit_code == 0) {
       exit_code = exec_as_user(working_dir, argv[optind]);
     }

http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/native/worker-launcher/impl/worker-launcher.c
----------------------------------------------------------------------
diff --git a/storm-core/src/native/worker-launcher/impl/worker-launcher.c b/storm-core/src/native/worker-launcher/impl/worker-launcher.c
index 5dc90d3..7baec01 100644
--- a/storm-core/src/native/worker-launcher/impl/worker-launcher.c
+++ b/storm-core/src/native/worker-launcher/impl/worker-launcher.c
@@ -414,33 +414,45 @@ static int copy_file(int input, const char* in_filename,
   return 0;
 }
 
-int setup_stormdist(FTSENT* entry, uid_t euser) {
+/**
+ * Sets up permissions for a directory optionally making it user-writable.
+ * We set up the permissions r(w)xrws--- so that the file group (should be Storm's user group)
+ * has complete access to the directory, and the file user (The topology owner's user)
+ * is able to read and execute, and in certain directories, write. The setGID bit is set
+ * to make sure any files created under the directory will be accessible to storm's user for
+ * cleanup purposes.
+ */
+static int setup_permissions(FTSENT* entry, uid_t euser, int user_write) {
   if (lchown(entry->fts_path, euser, launcher_gid) != 0) {
-    fprintf(ERRORFILE, "Failure to exec app initialization process - %s\n",
-      strerror(errno));
+    fprintf(ERRORFILE, "Failure to exec app initialization process - %s, fts_path=%s\n",
+            strerror(errno), entry->fts_path);
      return -1;
   }
   mode_t mode = entry->fts_statp->st_mode;
-  mode_t new_mode = (mode & (S_IRWXU)) | S_IRGRP | S_IWGRP;
-  if ((mode & S_IXUSR) == S_IXUSR) {
-    new_mode = new_mode | S_IXGRP;
+  // Preserve user read and execute and set group read and write.
+  mode_t new_mode = (mode & (S_IRUSR | S_IXUSR)) | S_IRGRP | S_IWGRP;
+  if (user_write) {
+    new_mode = new_mode | S_IWUSR;
   }
+  // If the entry is a directory, Add group execute and setGID bits.
   if ((mode & S_IFDIR) == S_IFDIR) {
-    new_mode = new_mode | S_ISGID;
+    new_mode = new_mode | S_IXGRP | S_ISGID;
   }
   if (chmod(entry->fts_path, new_mode) != 0) {
-    fprintf(ERRORFILE, "Failure to exec app initialization process - %s\n",
-      strerror(errno));
+    fprintf(ERRORFILE, "Failure to exec app initialization process - %s, fts_path=%s\n",
+            strerror(errno), entry->fts_path);
     return -1;
   }
   return 0;
 }
 
-int setup_stormdist_dir(const char* local_dir) {
+
+int setup_dir_permissions(const char* local_dir, int user_writable) {
   //This is the same as
   //> chmod g+rwX -R $local_dir
-  //> chown -no-dereference -R $user:$supervisor-group $local_dir 
-
+  //> chmod g+s -R $local_dir
+  //> if [ $user_writable ]; then chmod u+w;  else u-w; fi
+  //> chown -no-dereference -R $user:$supervisor-group $local_dir
   int exit_code = 0;
   uid_t euser = geteuid();
 
@@ -450,7 +462,7 @@ int setup_stormdist_dir(const char* local_dir) {
   } else {
     char *(paths[]) = {strndup(local_dir,PATH_MAX), 0};
     if (paths[0] == NULL) {
-      fprintf(ERRORFILE, "Malloc failed in setup_stormdist_dir\n");
+      fprintf(ERRORFILE, "Malloc failed in setup_dir_permissions\n");
       return -1;
     }
     // check to make sure the directory exists
@@ -485,14 +497,14 @@ int setup_stormdist_dir(const char* local_dir) {
 
       case FTS_DP:        // A directory being visited in post-order
       case FTS_DOT:       // A dot directory
+      case FTS_SL:        // A symbolic link
+      case FTS_SLNONE:    // A broken symbolic link
         //NOOP
         fprintf(LOGFILE, "NOOP: %s\n", entry->fts_path); break;
       case FTS_D:         // A directory in pre-order
       case FTS_F:         // A regular file
-      case FTS_SL:        // A symbolic link
-      case FTS_SLNONE:    // A broken symbolic link
-        if (setup_stormdist(entry, euser) != 0) {
-          exit_code = -1;
+        if (setup_permissions(entry, euser, user_writable) != 0) {
+            exit_code = -1;
         }
         break;
       case FTS_DEFAULT:   // Unknown type of file

http://git-wip-us.apache.org/repos/asf/storm/blob/81cd0745/storm-core/src/native/worker-launcher/impl/worker-launcher.h
----------------------------------------------------------------------
diff --git a/storm-core/src/native/worker-launcher/impl/worker-launcher.h b/storm-core/src/native/worker-launcher/impl/worker-launcher.h
index 3b1ec24..4a0b657 100644
--- a/storm-core/src/native/worker-launcher/impl/worker-launcher.h
+++ b/storm-core/src/native/worker-launcher/impl/worker-launcher.h
@@ -66,7 +66,7 @@ extern FILE *LOGFILE;
 // the log file for error messages
 extern FILE *ERRORFILE;
 
-int setup_stormdist_dir(const char* local_dir);
+int setup_dir_permissions(const char* local_dir, int for_blob_permission);
 
 int exec_as_user(const char * working_dir, const char * args);
 


[10/21] storm git commit: STORM-2018: Move back to JDK 1.7

Posted by ka...@apache.org.
STORM-2018: Move back to JDK 1.7


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/24ff529b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/24ff529b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/24ff529b

Branch: refs/heads/1.x-branch
Commit: 24ff529b8b3fcb99956a897aacfd96f4b85bba7d
Parents: 318ab5f
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Wed Sep 21 10:04:50 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Wed Sep 21 10:04:50 2016 -0500

----------------------------------------------------------------------
 pom.xml                                         |  4 +--
 .../storm/daemon/supervisor/BasicContainer.java | 15 +++++---
 .../daemon/supervisor/ReadClusterState.java     |  2 +-
 .../storm/daemon/supervisor/Supervisor.java     |  7 +++-
 .../daemon/supervisor/BasicContainerTest.java   | 37 +++++++++++---------
 .../storm/daemon/supervisor/ContainerTest.java  |  2 +-
 .../storm/localizer/AsyncLocalizerTest.java     |  9 +++--
 7 files changed, 48 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0665c10..a1466c9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1067,8 +1067,8 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
+                    <source>1.7</source>
+                    <target>1.7</target>
                 </configuration>
             </plugin>
             <plugin>

http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
index 237d89b..c5b40f2 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -52,7 +52,12 @@ import com.google.common.collect.Lists;
  */
 public class BasicContainer extends Container {
     private static final Logger LOG = LoggerFactory.getLogger(BasicContainer.class);
-    private static final FilenameFilter jarFilter = (dir, name) -> name.endsWith(".jar");
+    private static final FilenameFilter jarFilter = new FilenameFilter() {
+        @Override
+        public boolean accept(File dir, String name) {
+            return name.endsWith(".jar");
+        }
+    };
     private static final Joiner CPJ = 
             Joiner.on(Utils.CLASS_PATH_SEPARATOR).skipNulls();
     
@@ -334,9 +339,11 @@ public class BasicContainer extends Container {
         if (files == null) {
             return Collections.emptyList();
         }
-
-        return Arrays.stream(files).map(f -> f.getAbsolutePath())
-                .collect(Collectors.toList());
+        ArrayList<String> ret = new ArrayList<>(files.length);
+        for (File f: files) {
+            ret.add(f.getAbsolutePath());
+        }
+        return ret;
     }
     
     protected List<String> frameworkClasspath() {

http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
index 27f18b6..40b4a93 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/ReadClusterState.java
@@ -69,7 +69,7 @@ public class ReadClusterState implements Runnable, AutoCloseable {
         this.superConf = supervisor.getConf();
         this.stormClusterState = supervisor.getStormClusterState();
         this.syncSupEventManager = supervisor.getEventManger();
-        this.assignmentVersions = new AtomicReference<>(new HashMap<>());
+        this.assignmentVersions = new AtomicReference<Map<String, VersionedData<Assignment>>>(new HashMap<String, VersionedData<Assignment>>());
         this.assignmentId = supervisor.getAssignmentId();
         this.iSuper = supervisor.getiSupervisor();
         this.localizer = supervisor.getAsyncLocalizer();

http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
index 5e89eea..1399e8d 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
@@ -241,7 +241,12 @@ public class Supervisor implements DaemonCommon, AutoCloseable {
                 throw new IllegalArgumentException("Cannot start server in local mode!");
             }
             launch();
-            Utils.addShutdownHookWithForceKillIn1Sec(() -> {this.close();});
+            Utils.addShutdownHookWithForceKillIn1Sec(new Runnable(){ 
+                @Override
+                public void run() {
+                    close();
+                }
+            });
             registerWorkerNumGauge("supervisor:num-slots-used-gauge", conf);
             StormMetricsRegistry.startMetricsReporters(conf);
         } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
index edf534a..38081f4 100644
--- a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
@@ -31,9 +31,12 @@ import java.util.Map;
 
 import org.apache.storm.Config;
 import org.apache.storm.daemon.supervisor.Container.ContainerType;
+import org.apache.storm.generated.Bolt;
 import org.apache.storm.generated.LocalAssignment;
 import org.apache.storm.generated.ProfileAction;
 import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.generated.SpoutSpec;
+import org.apache.storm.generated.StateSpoutSpec;
 import org.apache.storm.generated.StormTopology;
 import org.apache.storm.utils.LocalState;
 import org.apache.storm.utils.Utils;
@@ -126,7 +129,7 @@ public class BasicContainerTest {
         LocalState ls = mock(LocalState.class);
         
         MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, 
-                "SUPERVISOR", port, la, ls, null, new HashMap<>(), ops, "profile");
+                "SUPERVISOR", port, la, ls, null, new HashMap<String, Object>(), ops, "profile");
         //null worker id means generate one...
         
         assertNotNull(mc._workerId);
@@ -155,7 +158,7 @@ public class BasicContainerTest {
         when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
         
         MockBasicContainer mc = new MockBasicContainer(ContainerType.RECOVER_FULL, superConf, 
-                "SUPERVISOR", port, la, ls, null, new HashMap<>(), ops, "profile");
+                "SUPERVISOR", port, la, ls, null, new HashMap<String, Object>(), ops, "profile");
         
         assertEquals(workerId, mc._workerId);
     }
@@ -175,7 +178,7 @@ public class BasicContainerTest {
         
         try {
             new MockBasicContainer(ContainerType.RECOVER_FULL, new HashMap<String, Object>(), 
-                    "SUPERVISOR", port, la, ls, null, new HashMap<>(), null, "profile");
+                    "SUPERVISOR", port, la, ls, null, new HashMap<String, Object>(), null, "profile");
             fail("Container recovered worker incorrectly");
         } catch (ContainerRecoveryException e) {
             //Expected
@@ -201,7 +204,7 @@ public class BasicContainerTest {
         when(ls.getApprovedWorkers()).thenReturn(new HashMap<>(workerState));
         
         MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, 
-                "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+                "SUPERVISOR", port, la, ls, workerId, new HashMap<String, Object>(), ops, "profile");
         
         mc.cleanUp();
         
@@ -235,7 +238,7 @@ public class BasicContainerTest {
         LocalState ls = mock(LocalState.class);
         
         MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, 
-                "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+                "SUPERVISOR", port, la, ls, workerId, new HashMap<String, Object>(), ops, "profile");
         
         //HEAP DUMP
         ProfileRequest req = new ProfileRequest();
@@ -369,9 +372,9 @@ public class BasicContainerTest {
         final String workerTmpDir = ContainerTest.asAbsPath(workerRoot, "tmp");
         
         final StormTopology st = new StormTopology();
-        st.set_spouts(new HashMap<>());
-        st.set_bolts(new HashMap<>());
-        st.set_state_spouts(new HashMap<>());
+        st.set_spouts(new HashMap<String, SpoutSpec>());
+        st.set_bolts(new HashMap<String, Bolt>());
+        st.set_state_spouts(new HashMap<String, StateSpoutSpec>());
         byte [] serializedState = Utils.gzip(Utils.thriftSerialize(st));
         
         final Map<String, Object> superConf = new HashMap<>();
@@ -379,19 +382,21 @@ public class BasicContainerTest {
         superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
         superConf.put(Config.STORM_LOG4J2_CONF_DIR, log4jdir);
         
-        LocalAssignment la = new LocalAssignment();
+        final LocalAssignment la = new LocalAssignment();
         la.set_topology_id(topoId);
         
-        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        final AdvancedFSOps ops = mock(AdvancedFSOps.class);
         when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
         when(ops.slurp(stormcode)).thenReturn(serializedState);
         
-        LocalState ls = mock(LocalState.class);
+        final LocalState ls = mock(LocalState.class);
         
         
-        checkpoint(() -> {
+        checkpoint(new Run() {
+          @Override
+          public void run() throws Exception {
             MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, 
-                    "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+                    "SUPERVISOR", port, la, ls, workerId, new HashMap<String, Object>(), ops, "profile");
 
             mc.launch();
 
@@ -440,7 +445,7 @@ public class BasicContainerTest {
                     workerId
                     ), cmd.cmd);
             assertEquals(new File(workerRoot), cmd.pwd);
-          }, 
+          }}, 
           "storm.home", stormHome,
           "storm.log.dir", stormLogDir);
     }
@@ -463,7 +468,7 @@ public class BasicContainerTest {
         LocalState ls = mock(LocalState.class);
         
         MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, 
-                "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+                "SUPERVISOR", port, la, ls, workerId, new HashMap<String, Object>(), ops, "profile");
         
         assertListEquals(Arrays.asList(
                 "-Xloggc:/tmp/storm/logs/gc.worker-9999-s-01-w-01-9999.log",
@@ -477,7 +482,7 @@ public class BasicContainerTest {
                 "-Xmx512m"),
                 mc.substituteChildopts(Arrays.asList("-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log","-Xms256m","-Xmx%HEAP-MEM%m"), memOnheap));
         
-        assertListEquals(Collections.emptyList(), 
+        assertListEquals(Collections.<String>emptyList(), 
                 mc.substituteChildopts(null));
     }
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
index 9b20ef8..3f34552 100644
--- a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/ContainerTest.java
@@ -101,7 +101,7 @@ public class ContainerTest {
         LocalAssignment la = new LocalAssignment();
         la.set_topology_id(topoId);
         MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf, 
-                "SUPERVISOR", 8080, la, "worker", new HashMap<>(), ops);
+                "SUPERVISOR", 8080, la, "worker", new HashMap<String, Object>(), ops);
         mc.kill();
         assertEquals(Collections.EMPTY_LIST, mc.killedPids);
         assertEquals(Collections.EMPTY_LIST, mc.forceKilledPids);

http://git-wip-us.apache.org/repos/asf/storm/blob/24ff529b/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java b/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
index 41c17c8..ed2b4ff 100644
--- a/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/localizer/AsyncLocalizerTest.java
@@ -34,8 +34,11 @@ import org.junit.Test;
 
 import org.apache.storm.Config;
 import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.generated.Bolt;
 import org.apache.storm.generated.ExecutorInfo;
 import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.SpoutSpec;
+import org.apache.storm.generated.StateSpoutSpec;
 import org.apache.storm.generated.StormTopology;
 import org.apache.storm.localizer.LocalizedResource;
 import org.apache.storm.localizer.Localizer;
@@ -128,9 +131,9 @@ public class AsyncLocalizerTest {
         final String simpleCurrentLocalFile = localizerRoot + user + "/simple.current";
        
         final StormTopology st = new StormTopology();
-        st.set_spouts(new HashMap<>());
-        st.set_bolts(new HashMap<>());
-        st.set_state_spouts(new HashMap<>());
+        st.set_spouts(new HashMap<String, SpoutSpec>());
+        st.set_bolts(new HashMap<String, Bolt>());
+        st.set_state_spouts(new HashMap<String, StateSpoutSpec>());
  
         Map<String, Map<String, Object>> topoBlobMap = new HashMap<>();
         Map<String, Object> simple = new HashMap<>();


[03/21] storm git commit: STORM-2018: Supervisor V2

Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/Utils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/Utils.java b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
index 7a322ec..9323db4 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/Utils.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/Utils.java
@@ -17,26 +17,16 @@
  */
 package org.apache.storm.utils;
 
-import org.apache.storm.Config;
-import org.apache.storm.blobstore.BlobStore;
-import org.apache.storm.blobstore.BlobStoreAclHandler;
-import org.apache.storm.blobstore.ClientBlobStore;
-import org.apache.storm.blobstore.InputStreamWithMeta;
-import org.apache.storm.blobstore.LocalFsBlobStore;
-import org.apache.storm.daemon.JarTransformer;
-import org.apache.storm.generated.*;
-import org.apache.storm.localizer.Localizer;
-import org.apache.storm.nimbus.NimbusInfo;
-import org.apache.storm.serialization.DefaultSerializationDelegate;
-import org.apache.storm.serialization.SerializationDelegate;
-
-import clojure.lang.IFn;
-import clojure.lang.RT;
-
+import clojure.lang.Keyword;
 import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.exec.CommandLine;
+import org.apache.commons.exec.DefaultExecutor;
+import org.apache.commons.exec.ExecuteException;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.input.ClassLoaderObjectInputStream;
 import org.apache.commons.lang.StringUtils;
 import org.apache.curator.ensemble.exhibitor.DefaultExhibitorRestClient;
@@ -44,6 +34,31 @@ import org.apache.curator.ensemble.exhibitor.ExhibitorEnsembleProvider;
 import org.apache.curator.ensemble.exhibitor.Exhibitors;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.storm.Config;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.blobstore.BlobStoreAclHandler;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.blobstore.InputStreamWithMeta;
+import org.apache.storm.blobstore.LocalFsBlobStore;
+import org.apache.storm.daemon.JarTransformer;
+import org.apache.storm.generated.AccessControl;
+import org.apache.storm.generated.AccessControlType;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.ClusterSummary;
+import org.apache.storm.generated.ComponentCommon;
+import org.apache.storm.generated.ComponentObject;
+import org.apache.storm.generated.GlobalStreamId;
+import org.apache.storm.generated.KeyNotFoundException;
+import org.apache.storm.generated.Nimbus;
+import org.apache.storm.generated.ReadableBlobMeta;
+import org.apache.storm.generated.SettableBlobMeta;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.generated.TopologyInfo;
+import org.apache.storm.generated.TopologySummary;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.nimbus.NimbusInfo;
+import org.apache.storm.serialization.DefaultSerializationDelegate;
+import org.apache.storm.serialization.SerializationDelegate;
 import org.apache.thrift.TBase;
 import org.apache.thrift.TDeserializer;
 import org.apache.thrift.TException;
@@ -61,12 +76,15 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.FileReader;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -77,13 +95,16 @@ import java.io.OutputStreamWriter;
 import java.io.PrintStream;
 import java.io.RandomAccessFile;
 import java.io.Serializable;
+import java.lang.management.ManagementFactory;
 import java.net.InetAddress;
 import java.net.URL;
 import java.net.URLDecoder;
 import java.net.UnknownHostException;
+import java.net.ServerSocket;
 import java.nio.ByteBuffer;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.attribute.PosixFilePermission;
 import java.util.ArrayList;
@@ -94,11 +115,14 @@ import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 import java.util.regex.Matcher;
@@ -108,8 +132,27 @@ import java.util.zip.GZIPOutputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
+import clojure.lang.RT;
+
 public class Utils {
-    private static final Logger LOG = LoggerFactory.getLogger(Utils.class);
+    // A singleton instance allows us to mock delegated static methods in our
+    // tests by subclassing.
+    private static Utils _instance = new Utils();
+
+    /**
+     * Provide an instance of this class for delegates to use.  To mock out
+     * delegated methods, provide an instance of a subclass that overrides the
+     * implementation of the delegated method.
+     * @param u a Utils instance
+     * @return the previously set instance
+     */
+    public static Utils setInstance(Utils u) {
+        Utils oldInstance = _instance;
+        _instance = u;
+        return oldInstance;
+    }
+
+    public static final Logger LOG = LoggerFactory.getLogger(Utils.class);
     public static final String DEFAULT_STREAM_ID = "default";
     public static final String DEFAULT_BLOB_VERSION_SUFFIX = ".version";
     public static final String CURRENT_BLOB_SUFFIX_ID = "current";
@@ -120,15 +163,35 @@ public class Utils {
     private static SerializationDelegate serializationDelegate;
     private static ClassLoader cl = null;
 
+    public static final boolean IS_ON_WINDOWS = "Windows_NT".equals(System.getenv("OS"));
+    public static final String FILE_PATH_SEPARATOR = System.getProperty("file.separator");
+    public static final String CLASS_PATH_SEPARATOR = System.getProperty("path.separator");
+
+    public static final int SIGKILL = 9;
+    public static final int SIGTERM = 15;
+
     static {
-        Map conf = readStormConfig();
+        Map<String, Object> conf = readStormConfig();
         serializationDelegate = getSerializationDelegate(conf);
     }
 
-    public static Object newInstance(String klass) {
+    @SuppressWarnings("unchecked")
+    public static <T> T newInstance(String klass) {
+        try {
+            return newInstance((Class<T>)Class.forName(klass));
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static <T> T newInstance(Class<T> klass) {
+        return _instance.newInstanceImpl(klass);
+    }
+
+    // Non-static impl methods exist for mocking purposes.
+    public <T> T newInstanceImpl(Class<T> klass) {
         try {
-            Class c = Class.forName(klass);
-            return c.newInstance();
+            return klass.newInstance();
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
@@ -150,11 +213,11 @@ public class Utils {
         return serializationDelegate.deserialize(serialized, clazz);
     }
 
-    public static <T> T thriftDeserialize(Class c, byte[] b, int offset, int length) {
+    public static <T> T thriftDeserialize(Class<T> c, byte[] b, int offset, int length) {
         try {
-            T ret = (T) c.newInstance();
+            T ret = c.newInstance();
             TDeserializer des = getDes();
-            des.deserialize((TBase)ret, b, offset, length);
+            des.deserialize((TBase) ret, b, offset, length);
             return ret;
         } catch (Exception e) {
             throw new RuntimeException(e);
@@ -261,7 +324,6 @@ public class Utils {
         return ret.toString();
     }
 
-
     public static long bitXorVals(List<Long> coll) {
         long result = 0;
         for (Long val : coll) {
@@ -291,16 +353,17 @@ public class Utils {
         }
     }
 
-    public static Map findAndReadConfigFile(String name, boolean mustExist) {
+    public static Map<String, Object> findAndReadConfigFile(String name, boolean mustExist) {
         InputStream in = null;
         boolean confFileEmpty = false;
         try {
             in = getConfigFileInputStream(name);
             if (null != in) {
                 Yaml yaml = new Yaml(new SafeConstructor());
-                Map ret = (Map) yaml.load(new InputStreamReader(in));
+                @SuppressWarnings("unchecked")
+                Map<String, Object> ret = (Map<String, Object>) yaml.load(new InputStreamReader(in));
                 if (null != ret) {
-                    return new HashMap(ret);
+                    return new HashMap<>(ret);
                 } else {
                     confFileEmpty = true;
                 }
@@ -312,7 +375,7 @@ public class Utils {
                 else
                     throw new RuntimeException("Could not find config file on classpath " + name);
             } else {
-                return new HashMap();
+                return new HashMap<>();
             }
         } catch (IOException e) {
             throw new RuntimeException(e);
@@ -354,16 +417,16 @@ public class Utils {
     }
 
 
-    public static Map findAndReadConfigFile(String name) {
+    public static Map<String, Object> findAndReadConfigFile(String name) {
         return findAndReadConfigFile(name, true);
     }
 
-    public static Map readDefaultConfig() {
+    public static Map<String, Object> readDefaultConfig() {
         return findAndReadConfigFile("defaults.yaml", true);
     }
 
-    public static Map readCommandLineOpts() {
-        Map ret = new HashMap();
+    public static Map<String, Object> readCommandLineOpts() {
+        Map<String, Object> ret = new HashMap<>();
         String commandOptions = System.getProperty("storm.options");
         if (commandOptions != null) {
             /*
@@ -394,10 +457,10 @@ public class Utils {
         return ret;
     }
 
-    public static Map readStormConfig() {
-        Map ret = readDefaultConfig();
+    public static Map<String, Object> readStormConfig() {
+        Map<String, Object> ret = readDefaultConfig();
         String confFile = System.getProperty("storm.conf.file");
-        Map storm;
+        Map<String, Object> storm;
         if (confFile == null || confFile.equals("")) {
             storm = findAndReadConfigFile("storm.yaml", false);
         } else {
@@ -487,7 +550,11 @@ public class Utils {
         HashMap nconf = new HashMap(conf);
         // only enable cleanup of blobstore on nimbus
         nconf.put(Config.BLOBSTORE_CLEANUP_ENABLE, Boolean.TRUE);
-        store.prepare(nconf, baseDir, nimbusInfo);
+
+        if(store != null) {
+            // store can be null during testing when mocking utils.
+            store.prepare(nconf, baseDir, nimbusInfo);
+        }
         return store;
     }
 
@@ -502,6 +569,11 @@ public class Utils {
      */
     public static void downloadResourcesAsSupervisor(String key, String localFile,
                                                      ClientBlobStore cb) throws AuthorizationException, KeyNotFoundException, IOException {
+        _instance.downloadResourcesAsSupervisorImpl(key, localFile, cb);
+    }
+
+    public void downloadResourcesAsSupervisorImpl(String key, String localFile,
+            ClientBlobStore cb) throws AuthorizationException, KeyNotFoundException, IOException {
         final int MAX_RETRY_ATTEMPTS = 2;
         final int ATTEMPTS_INTERVAL_TIME = 100;
         for (int retryAttempts = 0; retryAttempts < MAX_RETRY_ATTEMPTS; retryAttempts++) {
@@ -520,11 +592,8 @@ public class Utils {
 
     private static boolean downloadResourcesAsSupervisorAttempt(ClientBlobStore cb, String key, String localFile) {
         boolean isSuccess = false;
-        FileOutputStream out = null;
-        InputStreamWithMeta in = null;
-        try {
-            out = new FileOutputStream(localFile);
-            in = cb.getBlob(key);
+        try (FileOutputStream out = new FileOutputStream(localFile);
+                InputStreamWithMeta in = cb.getBlob(key);) {
             long fileSize = in.getFileLength();
 
             byte[] buffer = new byte[1024];
@@ -538,17 +607,6 @@ public class Utils {
             isSuccess = (fileSize == downloadFileSize);
         } catch (TException | IOException e) {
             LOG.error("An exception happened while downloading {} from blob store.", localFile, e);
-        } finally {
-            try {
-                if (out != null) {
-                    out.close();
-                }
-            } catch (IOException ignored) {}
-            try {
-                if (in != null) {
-                    in.close();
-                }
-            } catch (IOException ignored) {}
         }
         if (!isSuccess) {
             try {
@@ -560,8 +618,21 @@ public class Utils {
         return isSuccess;
     }
 
+    public static boolean checkFileExists(File path) {
+        return Files.exists(path.toPath());
+    }
+    
+    public static boolean checkFileExists(String path) {
+        return Files.exists(new File(path).toPath());
+    }
+
     public static boolean checkFileExists(String dir, String file) {
-        return Files.exists(new File(dir, file).toPath());
+        return checkFileExists(dir + FILE_PATH_SEPARATOR + file);
+    }
+
+    public static boolean CheckDirExists(String dir) {
+        File file = new File(dir);
+        return file.isDirectory();
     }
 
     public static long nimbusVersionOfBlob(String key, ClientBlobStore cb) throws AuthorizationException, KeyNotFoundException {
@@ -626,27 +697,19 @@ public class Utils {
     }
 
 
-    public static synchronized IFn loadClojureFn(String namespace, String name) {
+    public static synchronized clojure.lang.IFn loadClojureFn(String namespace, String name) {
         try {
             clojure.lang.Compiler.eval(RT.readString("(require '" + namespace + ")"));
         } catch (Exception e) {
             //if playing from the repl and defining functions, file won't exist
         }
-        return (IFn) RT.var(namespace, name).deref();
+        return (clojure.lang.IFn) RT.var(namespace, name).deref();
     }
 
     public static boolean isSystemId(String id) {
         return id.startsWith("__");
     }
 
-    public static <K, V> Map<V, K> reverseMap(Map<K, V> map) {
-        Map<V, K> ret = new HashMap<V, K>();
-        for (Map.Entry<K, V> entry : map.entrySet()) {
-            ret.put(entry.getValue(), entry.getKey());
-        }
-        return ret;
-    }
-
     public static ComponentCommon getComponentCommon(StormTopology topology, String id) {
         if (topology.get_spouts().containsKey(id)) {
             return topology.get_spouts().get(id).get_common();
@@ -714,7 +777,7 @@ public class Utils {
         }
     }
 
-    public static <T> T thriftDeserialize(Class c, byte[] b) {
+    public static <T> T thriftDeserialize(Class<T> c, byte[] b) {
         try {
             return Utils.thriftDeserialize(c, b, 0, b.length);
         } catch (Exception e) {
@@ -874,7 +937,7 @@ public class Utils {
         }
 
         boolean gzipped = inFile.toString().endsWith("gz");
-        if (onWindows()) {
+        if (isOnWindows()) {
             // Tar is not native to Windows. Use simple Java based implementation for
             // tests and simple tar archives
             unTarUsingJava(inFile, untarDir, gzipped);
@@ -916,7 +979,6 @@ public class Utils {
     private static void unTarUsingJava(File inFile, File untarDir,
                                        boolean gzipped) throws IOException {
         InputStream inputStream = null;
-        TarArchiveInputStream tis = null;
         try {
             if (gzipped) {
                 inputStream = new BufferedInputStream(new GZIPInputStream(
@@ -924,31 +986,15 @@ public class Utils {
             } else {
                 inputStream = new BufferedInputStream(new FileInputStream(inFile));
             }
-            tis = new TarArchiveInputStream(inputStream);
-            for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null; ) {
-                unpackEntries(tis, entry, untarDir);
-                entry = tis.getNextTarEntry();
+            try (TarArchiveInputStream tis = new TarArchiveInputStream(inputStream)) {
+                for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null; ) {
+                    unpackEntries(tis, entry, untarDir);
+                    entry = tis.getNextTarEntry();
+                }
             }
         } finally {
-            cleanup(tis, inputStream);
-        }
-    }
-
-    /**
-     * Close the Closeable objects and <b>ignore</b> any {@link IOException} or
-     * null pointers. Must only be used for cleanup in exception handlers.
-     *
-     * @param closeables the objects to close
-     */
-    private static void cleanup(java.io.Closeable... closeables) {
-        for (java.io.Closeable c : closeables) {
-            if (c != null) {
-                try {
-                    c.close();
-                } catch (IOException e) {
-                    LOG.debug("Exception in closing " + c, e);
-
-                }
+            if(inputStream != null) {
+                inputStream.close();
             }
         }
     }
@@ -970,7 +1016,7 @@ public class Utils {
         if (!outputFile.getParentFile().exists()) {
             if (!outputFile.getParentFile().mkdirs()) {
                 throw new IOException("Mkdirs failed to create tar internal dir "
-                        + outputDir);
+                                      + outputDir);
             }
         }
         int count;
@@ -985,13 +1031,17 @@ public class Utils {
         outputStream.close();
     }
 
-    public static boolean onWindows() {
+    public static boolean isOnWindows() {
         if (System.getenv("OS") != null) {
             return System.getenv("OS").equals("Windows_NT");
         }
         return false;
     }
 
+    public static boolean isAbsolutePath(String path) {
+        return Paths.get(path).isAbsolute();
+    }
+
     public static void unpack(File localrsrc, File dst) throws IOException {
         String lowerDst = localrsrc.getName().toLowerCase();
         if (lowerDst.endsWith(".jar")) {
@@ -1031,6 +1081,10 @@ public class Utils {
         return newCurator(conf, servers, port, root, null);
     }
 
+    public static CuratorFramework newCurator(Map conf, List<String> servers, Object port, ZookeeperAuthInfo auth) {
+        return newCurator(conf, servers, port, "", auth);
+    }
+
     public static CuratorFramework newCurator(Map conf, List<String> servers, Object port, String root, ZookeeperAuthInfo auth) {
         List<String> serverPorts = new ArrayList<String>();
         for (String zkServer : servers) {
@@ -1071,17 +1125,19 @@ public class Utils {
             .connectionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT)))
             .sessionTimeoutMs(Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)))
             .retryPolicy(new StormBoundedExponentialBackoffRetry(
-                        Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)),
-                        Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)),
-                        Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES))));
+                    Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)),
+                    Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)),
+                    Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES))));
 
         if (auth != null && auth.scheme != null && auth.payload != null) {
             builder.authorization(auth.scheme, auth.payload);
         }
     }
 
-    public static CuratorFramework newCurator(Map conf, List<String> servers, Object port, ZookeeperAuthInfo auth) {
-        return newCurator(conf, servers, port, "", auth);
+    public static void testSetupBuilder(CuratorFrameworkFactory.Builder
+                                                builder, String zkStr, Map conf, ZookeeperAuthInfo auth)
+    {
+        setupBuilder(builder, zkStr, conf, auth);
     }
 
     public static CuratorFramework newCuratorStarted(Map conf, List<String> servers, Object port, String root, ZookeeperAuthInfo auth) {
@@ -1122,10 +1178,16 @@ public class Utils {
                 LOG.info("{}:{}", prefix, line);
             }
         } catch (IOException e) {
-            LOG.warn("Error whiel trying to log stream", e);
+            LOG.warn("Error while trying to log stream", e);
         }
     }
 
+    /**
+     * Checks if a throwable is an instance of a particular class
+     * @param klass The class you're expecting
+     * @param throwable The throwable you expect to be an instance of klass
+     * @return true if throwable is instance of klass, false otherwise.
+     */
     public static boolean exceptionCauseIsInstanceOf(Class klass, Throwable throwable) {
         Throwable t = throwable;
         while (t != null) {
@@ -1161,6 +1223,7 @@ public class Utils {
                 && !((String)conf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME)).isEmpty());
     }
 
+
     public static List<ACL> getWorkerACL(Map conf) {
         //This is a work around to an issue with ZK where a sasl super user is not super unless there is an open SASL ACL so we are trying to give the correct perms
         if (!isZkAuthenticationConfiguredTopology(conf)) {
@@ -1168,11 +1231,11 @@ public class Utils {
         }
         String stormZKUser = (String)conf.get(Config.STORM_ZOOKEEPER_SUPERACL);
         if (stormZKUser == null) {
-            throw new IllegalArgumentException("Authentication is enabled but "+Config.STORM_ZOOKEEPER_SUPERACL+" is not set");
+            throw new IllegalArgumentException("Authentication is enabled but " + Config.STORM_ZOOKEEPER_SUPERACL + " is not set");
         }
-        String[] split = stormZKUser.split(":",2);
+        String[] split = stormZKUser.split(":", 2);
         if (split.length != 2) {
-            throw new IllegalArgumentException(Config.STORM_ZOOKEEPER_SUPERACL+" does not appear to be in the form scheme:acl, i.e. sasl:storm-user");
+            throw new IllegalArgumentException(Config.STORM_ZOOKEEPER_SUPERACL + " does not appear to be in the form scheme:acl, i.e. sasl:storm-user");
         }
         ArrayList<ACL> ret = new ArrayList<ACL>(ZooDefs.Ids.CREATOR_ALL_ACL);
         ret.add(new ACL(ZooDefs.Perms.ALL, new Id(split[0], split[1])));
@@ -1211,6 +1274,10 @@ public class Utils {
         }
     }
 
+    /**
+     * Gets some information, including stack trace, for a running thread.
+     * @return A human-readable string of the dump.
+     */
     public static String threadDump() {
         final StringBuilder dump = new StringBuilder();
         final java.lang.management.ThreadMXBean threadMXBean =  java.lang.management.ManagementFactory.getThreadMXBean();
@@ -1236,20 +1303,19 @@ public class Utils {
         return dump.toString();
     }
 
-    // Assumes caller is synchronizing
+    /**
+     * Creates an instance of the pluggable SerializationDelegate or falls back to
+     * DefaultSerializationDelegate if something goes wrong.
+     * @param stormConf The config from which to pull the name of the pluggable class.
+     * @return an instance of the class specified by storm.meta.serialization.delegate
+     */
     private static SerializationDelegate getSerializationDelegate(Map stormConf) {
         String delegateClassName = (String)stormConf.get(Config.STORM_META_SERIALIZATION_DELEGATE);
         SerializationDelegate delegate;
         try {
             Class delegateClass = Class.forName(delegateClassName);
             delegate = (SerializationDelegate) delegateClass.newInstance();
-        } catch (ClassNotFoundException e) {
-            LOG.error("Failed to construct serialization delegate, falling back to default", e);
-            delegate = new DefaultSerializationDelegate();
-        } catch (InstantiationException e) {
-            LOG.error("Failed to construct serialization delegate, falling back to default", e);
-            delegate = new DefaultSerializationDelegate();
-        } catch (IllegalAccessException e) {
+        } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) {
             LOG.error("Failed to construct serialization delegate, falling back to default", e);
             delegate = new DefaultSerializationDelegate();
         }
@@ -1295,7 +1361,7 @@ public class Utils {
                         if (!file.getParentFile().mkdirs()) {
                             if (!file.getParentFile().isDirectory()) {
                                 throw new IOException("Mkdirs failed to create " +
-                                        file.getParentFile().toString());
+                                                      file.getParentFile().toString());
                             }
                         }
                         OutputStream out = new FileOutputStream(file);
@@ -1358,13 +1424,16 @@ public class Utils {
             }
             if (memoryOpts != null) {
                 int unit = 1;
-                if (memoryOpts.toLowerCase().endsWith("k")) {
+                memoryOpts = memoryOpts.toLowerCase();
+
+                if (memoryOpts.endsWith("k")) {
                     unit = 1024;
-                } else if (memoryOpts.toLowerCase().endsWith("m")) {
+                } else if (memoryOpts.endsWith("m")) {
                     unit = 1024 * 1024;
-                } else if (memoryOpts.toLowerCase().endsWith("g")) {
+                } else if (memoryOpts.endsWith("g")) {
                     unit = 1024 * 1024 * 1024;
                 }
+
                 memoryOpts = memoryOpts.replaceAll("[a-zA-Z]", "");
                 Double result =  Double.parseDouble(memoryOpts) * unit / 1024.0 / 1024.0;
                 return (result < 1.0) ? 1.0 : result;
@@ -1387,18 +1456,29 @@ public class Utils {
     }
 
     public static TopologyInfo getTopologyInfo(String name, String asUser, Map stormConf) {
-        TopologyInfo topologyInfo = null;
         try (NimbusClient client = NimbusClient.getConfiguredClientAs(stormConf, asUser)) {
-            ClusterSummary summary = client.getClient().getClusterInfo();
+            String topologyId = getTopologyId(name, client.getClient());
+            if (null != topologyId) {
+                return client.getClient().getTopologyInfo(topologyId);
+            }
+            return null;
+        } catch(Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static String getTopologyId(String name, Nimbus.Client client) {
+        try {
+            ClusterSummary summary = client.getClusterInfo();
             for(TopologySummary s : summary.get_topologies()) {
                 if(s.get_name().equals(name)) {
-                    topologyInfo = client.getClient().getTopologyInfo(s.get_id());
+                    return s.get_id();
                 }
             }
         } catch(Exception e) {
             throw new RuntimeException(e);
         }
-        return topologyInfo;
+        return null;
     }
 
     /**
@@ -1414,8 +1494,744 @@ public class Utils {
         return number & Integer.MAX_VALUE;
     }
 
-    public static String localHostname() throws UnknownHostException {
+    public static GlobalStreamId getGlobalStreamId(String streamId, String componentId) {
+        if (componentId == null) {
+            return new GlobalStreamId(streamId, DEFAULT_STREAM_ID);
+        }
+        return new GlobalStreamId(streamId, componentId);
+    }
+
+    public static RuntimeException wrapInRuntime(Exception e){
+        if (e instanceof RuntimeException){
+            return (RuntimeException)e;
+        } else {
+            return new RuntimeException(e);
+        }
+    }
+
+    public static int getAvailablePort(int prefferedPort) {
+        int localPort = -1;
+        try(ServerSocket socket = new ServerSocket(prefferedPort)) {
+            localPort = socket.getLocalPort();
+        } catch(IOException exp) {
+            if (prefferedPort > 0) {
+                return getAvailablePort(0);
+            }
+        }
+        return localPort;
+    }
+
+    public static int getAvailablePort() {
+        return getAvailablePort(0);
+    }
+
+    /**
+     * Determines if a zip archive contains a particular directory.
+     *
+     * @param zipfile path to the zipped file
+     * @param target directory being looked for in the zip.
+     * @return boolean whether or not the directory exists in the zip.
+     */
+    public static boolean zipDoesContainDir(String zipfile, String target) throws IOException {
+        List<ZipEntry> entries = (List<ZipEntry>) Collections.list(new ZipFile(zipfile).entries());
+
+        String targetDir = target + "/";
+        for(ZipEntry entry : entries) {
+            String name = entry.getName();
+            if(name.startsWith(targetDir)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Joins any number of maps together into a single map, combining their values into
+     * a list, maintaining values in the order the maps were passed in. Nulls are inserted
+     * for given keys when the map does not contain that key.
+     *
+     * i.e. joinMaps({'a' => 1, 'b' => 2}, {'b' => 3}, {'a' => 4, 'c' => 5}) ->
+     *      {'a' => [1, null, 4], 'b' => [2, 3, null], 'c' => [null, null, 5]}
+     *
+     * @param maps variable number of maps to join - order affects order of values in output.
+     * @return combined map
+     */
+    public static <K, V> Map<K, List<V>> joinMaps(Map<K, V>... maps) {
+        Map<K, List<V>> ret = new HashMap<>();
+
+        Set<K> keys = new HashSet<>();
+
+        for(Map<K, V> map : maps) {
+            keys.addAll(map.keySet());
+        }
+
+        for(Map<K, V> m : maps) {
+            for(K key : keys) {
+                V value = m.get(key);
+
+                if(!ret.containsKey(key)) {
+                    ret.put(key, new ArrayList<V>());
+                }
+
+                List<V> targetList = ret.get(key);
+                targetList.add(value);
+            }
+        }
+        return ret;
+    }
+
+    /**
+     * Fills up chunks out of a collection (given a maximum amount of chunks)
+     *
+     * i.e. partitionFixed(5, [1,2,3]) -> [[1,2,3]]
+     *      partitionFixed(5, [1..9]) -> [[1,2], [3,4], [5,6], [7,8], [9]]
+     *      partitionFixed(3, [1..10]) -> [[1,2,3,4], [5,6,7], [8,9,10]]
+     * @param maxNumChunks the maximum number of chunks to return
+     * @param coll the collection to be chunked up
+     * @return a list of the chunks, which are themselves lists.
+     */
+    public static <T> List<List<T>> partitionFixed(int maxNumChunks, Collection<T> coll) {
+        List<List<T>> ret = new ArrayList<>();
+
+        if(maxNumChunks == 0 || coll == null) {
+            return ret;
+        }
+
+        Map<Integer, Integer> parts = integerDivided(coll.size(), maxNumChunks);
+
+        // Keys sorted in descending order
+        List<Integer> sortedKeys = new ArrayList<Integer>(parts.keySet());
+        Collections.sort(sortedKeys, Collections.reverseOrder());
+
+
+        Iterator<T> it = coll.iterator();
+        for(Integer chunkSize : sortedKeys) {
+            if(!it.hasNext()) { break; }
+            Integer times = parts.get(chunkSize);
+            for(int i = 0; i < times; i++) {
+                if(!it.hasNext()) { break; }
+                List<T> chunkList = new ArrayList<>();
+                for(int j = 0; j < chunkSize; j++) {
+                    if(!it.hasNext()) { break; }
+                    chunkList.add(it.next());
+                }
+                ret.add(chunkList);
+            }
+        }
+
+        return ret;
+    }
+
+    /**
+     * Return a new instance of a pluggable specified in the conf.
+     * @param conf The conf to read from.
+     * @param configKey The key pointing to the pluggable class
+     * @return an instance of the class or null if it is not specified.
+     */
+    public static Object getConfiguredClass(Map conf, Object configKey) {
+        if (conf.containsKey(configKey)) {
+            return newInstance((String)conf.get(configKey));
+        }
+        return null;
+    }
+
+    public static String logsFilename(String stormId, String port) {
+        return stormId + FILE_PATH_SEPARATOR + port + FILE_PATH_SEPARATOR + "worker.log";
+    }
+
+    public static String eventLogsFilename(String stormId, String port) {
+        return stormId + FILE_PATH_SEPARATOR + port + FILE_PATH_SEPARATOR + "events.log";
+    }
+
+    public static Object readYamlFile(String yamlFile) {
+        try (FileReader reader = new FileReader(yamlFile)) {
+            return new Yaml(new SafeConstructor()).load(reader);
+        } catch(Exception ex) {
+            LOG.error("Failed to read yaml file.", ex);
+        }
+        return null;
+    }
+
+    public static void setupDefaultUncaughtExceptionHandler() {
+        Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+                public void uncaughtException(Thread thread, Throwable thrown) {
+                    try {
+                        handleUncaughtException(thrown);
+                    } catch (Error err) {
+                        LOG.error("Received error in main thread.. terminating server...", err);
+                        Runtime.getRuntime().exit(-2);
+                    }
+                }
+            });
+    }
+
+    /**
+     * Creates a new map with a string value in the map replaced with an
+     * equivalently-lengthed string of '#'.
+     * @param m The map that a value will be redacted from
+     * @param key The key pointing to the value to be redacted
+     * @return a new map with the value redacted. The original map will not be modified.
+     */
+    public static Map<Object, String> redactValue(Map<Object, String> m, Object key) {
+        if(m.containsKey(key)) {
+            HashMap<Object, String> newMap = new HashMap<>(m);
+            String value = newMap.get(key);
+            String redacted = new String(new char[value.length()]).replace("\0", "#");
+            newMap.put(key, redacted);
+            return newMap;
+        }
+        return m;
+    }
+
+    /**
+     * Make sure a given key name is valid for the storm config.
+     * Throw RuntimeException if the key isn't valid.
+     * @param name The name of the config key to check.
+     */
+    private static final Set<String> disallowedKeys = new HashSet<>(Arrays.asList(new String[] {"/", ".", ":", "\\"}));
+    public static void validateKeyName(String name) {
+
+        for(String key : disallowedKeys) {
+            if( name.contains(key) ) {
+                throw new RuntimeException("Key name cannot contain any of the following: " + disallowedKeys.toString());
+            }
+        }
+        if(name.trim().isEmpty()) {
+            throw new RuntimeException("Key name cannot be blank");
+        }
+    }
+
+    public static String localHostname () throws UnknownHostException {
+        return _instance.localHostnameImpl();
+    }
+
+    // Non-static impl methods exist for mocking purposes.
+    protected String localHostnameImpl () throws UnknownHostException {
         return InetAddress.getLocalHost().getCanonicalHostName();
     }
-}
 
+    private static String memoizedLocalHostnameString = null;
+
+    public static String memoizedLocalHostname () throws UnknownHostException {
+        if (memoizedLocalHostnameString == null) {
+            memoizedLocalHostnameString = localHostname();
+        }
+        return memoizedLocalHostnameString;
+    }
+
+    /**
+     * Gets the storm.local.hostname value, or tries to figure out the local hostname
+     * if it is not set in the config.
+     * @param conf The storm config to read from
+     * @return a string representation of the hostname.
+    */
+    public static String hostname (Map<String, Object> conf) throws UnknownHostException  {
+        if (conf == null) {
+            return memoizedLocalHostname();
+        }
+        Object hostnameString = conf.get(Config.STORM_LOCAL_HOSTNAME);
+        if (hostnameString == null || hostnameString.equals("")) {
+            return memoizedLocalHostname();
+        }
+        return (String)hostnameString;
+    }
+
+    public static String uuid() {
+        return UUID.randomUUID().toString();
+    }
+
+    public static void exitProcess (int val, String msg) {
+        String combinedErrorMessage = "Halting process: " + msg;
+        LOG.error(combinedErrorMessage, new RuntimeException(combinedErrorMessage));
+        Runtime.getRuntime().exit(val);
+    }
+
+    public static Runnable mkSuicideFn() {
+        return new Runnable() {
+            @Override
+            public void run() {
+                Utils.exitProcess(1, "Worker died");
+            }
+        };
+    }
+
+    /**
+     * "{:a 1 :b 1 :c 2} -> {1 [:a :b] 2 :c}"
+     *
+     * Example usage in java:
+     *  Map<Integer, String> tasks;
+     *  Map<String, List<Integer>> componentTasks = Utils.reverse_map(tasks);
+     *
+     * The order of he resulting list values depends on the ordering properties
+     * of the Map passed in. The caller is responsible for passing an ordered
+     * map if they expect the result to be consistently ordered as well.
+     *
+     * @param map to reverse
+     * @return a reversed map
+     */
+    public static <K, V> HashMap<V, List<K>> reverseMap(Map<K, V> map) {
+        HashMap<V, List<K>> rtn = new HashMap<V, List<K>>();
+        if (map == null) {
+            return rtn;
+        }
+        for (Entry<K, V> entry : map.entrySet()) {
+            K key = entry.getKey();
+            V val = entry.getValue();
+            List<K> list = rtn.get(val);
+            if (list == null) {
+                list = new ArrayList<K>();
+                rtn.put(entry.getValue(), list);
+            }
+            list.add(key);
+        }
+        return rtn;
+    }
+
+    /**
+     * "[[:a 1] [:b 1] [:c 2]} -> {1 [:a :b] 2 :c}"
+     * Reverses an assoc-list style Map like reverseMap(Map...)
+     *
+     * @param listSeq to reverse
+     * @return a reversed map
+     */
+    public static HashMap reverseMap(List listSeq) {
+        HashMap<Object, List<Object>> rtn = new HashMap();
+        if (listSeq == null) {
+            return rtn;
+        }
+        for (Object entry : listSeq) {
+            List listEntry = (List) entry;
+            Object key = listEntry.get(0);
+            Object val = listEntry.get(1);
+            List list = rtn.get(val);
+            if (list == null) {
+                list = new ArrayList<Object>();
+                rtn.put(val, list);
+            }
+            list.add(key);
+        }
+        return rtn;
+    }
+
+
+    /**
+     * @return the pid of this JVM, because Java doesn't provide a real way to do this.
+     */
+    public static String processPid() {
+        String name = ManagementFactory.getRuntimeMXBean().getName();
+        String[] split = name.split("@");
+        if (split.length != 2) {
+            throw new RuntimeException("Got unexpected process name: " + name);
+        }
+        return split[0];
+    }
+
+    public static int execCommand(String... command) throws ExecuteException, IOException {
+        CommandLine cmd = new CommandLine(command[0]);
+        for (int i = 1; i < command.length; i++) {
+            cmd.addArgument(command[i]);
+        }
+
+        DefaultExecutor exec = new DefaultExecutor();
+        return exec.execute(cmd);
+    }
+
+    /**
+     * Extract dir from the jar to destdir
+     *
+     * @param jarpath Path to the jar file
+     * @param dir Directory in the jar to pull out
+     * @param destdir Path to the directory where the extracted directory will be put
+     */
+    public static void extractDirFromJar(String jarpath, String dir, File destdir) {
+        _instance.extractDirFromJarImpl(jarpath, dir, destdir);
+    }
+    
+    public void extractDirFromJarImpl(String jarpath, String dir, File destdir) {
+        try (JarFile jarFile = new JarFile(jarpath)) {
+            Enumeration<JarEntry> jarEnums = jarFile.entries();
+            while (jarEnums.hasMoreElements()) {
+                JarEntry entry = jarEnums.nextElement();
+                if (!entry.isDirectory() && entry.getName().startsWith(dir)) {
+                    File aFile = new File(destdir, entry.getName());
+                    aFile.getParentFile().mkdirs();
+                    try (FileOutputStream out = new FileOutputStream(aFile);
+                         InputStream in = jarFile.getInputStream(entry)) {
+                        IOUtils.copy(in, out);
+                    }
+                }
+            }
+        } catch (IOException e) {
+            LOG.info("Could not extract {} from {}", dir, jarpath);
+        }
+    }
+
+    public static void sendSignalToProcess(long lpid, int signum) throws IOException {
+        String pid = Long.toString(lpid);
+        try {
+            if (isOnWindows()) {
+                if (signum == SIGKILL) {
+                    execCommand("taskkill", "/f", "/pid", pid);
+                } else {
+                    execCommand("taskkill", "/pid", pid);
+                }
+            } else {
+                execCommand("kill", "-" + signum, pid);
+            }
+        } catch (ExecuteException e) {
+            LOG.info("Error when trying to kill {}. Process is probably already dead.", pid);
+        } catch (IOException e) {
+            LOG.info("IOException Error when trying to kill {}.", pid);
+            throw e;
+        }
+    }
+
+    public static void forceKillProcess (String pid) throws IOException {
+        sendSignalToProcess(Long.parseLong(pid), SIGKILL);
+    }
+
+    public static void killProcessWithSigTerm (String pid) throws IOException {
+        sendSignalToProcess(Long.parseLong(pid), SIGTERM);
+    }
+
+    /**
+     * Adds the user supplied function as a shutdown hook for cleanup.
+     * Also adds a function that sleeps for a second and then halts the
+     * runtime to avoid any zombie process in case cleanup function hangs.
+     */
+    public static void addShutdownHookWithForceKillIn1Sec (Runnable func) {
+        Runnable sleepKill = new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    Time.sleepSecs(1);
+                    Runtime.getRuntime().halt(20);
+                } catch (Exception e) {
+                    LOG.warn("Exception in the ShutDownHook", e);
+                }
+            }
+        };
+        Runtime.getRuntime().addShutdownHook(new Thread(func));
+        Runtime.getRuntime().addShutdownHook(new Thread(sleepKill));
+    }
+
+    /**
+     * Returns the combined string, escaped for posix shell.
+     * @param command the list of strings to be combined
+     * @return the resulting command string
+     */
+    public static String shellCmd (List<String> command) {
+        List<String> changedCommands = new ArrayList<>(command.size());
+        for (String str: command) {
+            if (str == null) {
+                continue;
+            }
+            changedCommands.add("'" + str.replaceAll("'", "'\"'\"'") + "'");
+        }
+        return StringUtils.join(changedCommands, " ");
+    }
+
+    public static String scriptFilePath (String dir) {
+        return dir + FILE_PATH_SEPARATOR + "storm-worker-script.sh";
+    }
+
+    public static String containerFilePath (String dir) {
+        return dir + FILE_PATH_SEPARATOR + "launch_container.sh";
+    }
+
+    public static Object nullToZero (Object v) {
+        return (v != null ? v : 0);
+    }
+
+    /**
+     * Deletes a file or directory and its contents if it exists. Does not
+     * complain if the input is null or does not exist.
+     * @param path the path to the file or directory
+     */
+    public static void forceDelete(String path) throws IOException {
+        _instance.forceDeleteImpl(path);
+    }
+
+    // Non-static impl methods exist for mocking purposes.
+    protected void forceDeleteImpl(String path) throws IOException {
+        LOG.debug("Deleting path {}", path);
+        if (checkFileExists(path)) {
+            try {
+                FileUtils.forceDelete(new File(path));
+            } catch (FileNotFoundException ignored) {}
+        }
+    }
+
+    /**
+     * Returns a Collection of file names found under the given directory.
+     * @param dir a directory
+     * @return the Collection of file names
+     */
+    public static Collection<String> readDirContents(String dir) {
+        Collection<String> ret = new HashSet<>();
+        File[] files = new File(dir).listFiles();
+        if (files != null) {
+            for (File f: files) {
+                ret.add(f.getName());
+            }
+        }
+        return ret;
+    }
+
+    /**
+     * Returns the value of java.class.path System property. Kept separate for
+     * testing.
+     * @return the classpath
+     */
+    public static String currentClasspath() {
+        return _instance.currentClasspathImpl();
+    }
+
+    // Non-static impl methods exist for mocking purposes.
+    public String currentClasspathImpl() {
+        return System.getProperty("java.class.path");
+    }
+
+    public static String addToClasspath(String classpath,
+                Collection<String> paths) {
+        return _instance.addToClasspathImpl(classpath, paths);
+    }
+
+    public static String addToClasspath(Collection<String> classpaths,
+                Collection<String> paths) {
+        return _instance.addToClasspathImpl(classpaths, paths);
+    }
+
+    // Non-static impl methods exist for mocking purposes.
+    public String addToClasspathImpl(String classpath,
+                Collection<String> paths) {
+        if (paths == null || paths.isEmpty()) {
+            return classpath;
+        }
+        List<String> l = new LinkedList<>();
+        l.add(classpath);
+        l.addAll(paths);
+        return StringUtils.join(l, CLASS_PATH_SEPARATOR);
+    }
+
+    public String addToClasspathImpl(Collection<String> classpaths,
+                Collection<String> paths) {
+        List<String> allPaths = new ArrayList<>();
+        if(classpaths != null) {
+            allPaths.addAll(classpaths);
+        }
+        if(paths != null) {
+            allPaths.addAll(paths);
+        }
+        return StringUtils.join(allPaths, CLASS_PATH_SEPARATOR);
+    }
+
+    public static class UptimeComputer {
+        int startTime = 0;
+
+        public UptimeComputer() {
+            startTime = Time.currentTimeSecs();
+        }
+
+        public int upTime() {
+            return Time.deltaSecs(startTime);
+        }
+    }
+
+    public static UptimeComputer makeUptimeComputer() {
+        return _instance.makeUptimeComputerImpl();
+    }
+
+    // Non-static impl methods exist for mocking purposes.
+    public UptimeComputer makeUptimeComputerImpl() {
+        return new UptimeComputer();
+    }
+
+    /**
+     * Writes a posix shell script file to be executed in its own process.
+     * @param dir the directory under which the script is to be written
+     * @param command the command the script is to execute
+     * @param environment optional environment variables to set before running the script's command. May be  null.
+     * @return the path to the script that has been written
+     */
+    public static String writeScript(String dir, List<String> command,
+                                     Map<String,String> environment) throws IOException {
+        String path = Utils.scriptFilePath(dir);
+        try(BufferedWriter out = new BufferedWriter(new FileWriter(path))) {
+            out.write("#!/bin/bash");
+            out.newLine();
+            if (environment != null) {
+                for (String k : environment.keySet()) {
+                    String v = environment.get(k);
+                    if (v == null) {
+                        v = "";
+                    }
+                    out.write(Utils.shellCmd(
+                            Arrays.asList(
+                                    "export",k+"="+v)));
+                    out.write(";");
+                    out.newLine();
+                }
+            }
+            out.newLine();
+            out.write("exec "+Utils.shellCmd(command)+";");
+        }
+        return path;
+    }
+
+    /**
+     * A thread that can answer if it is sleeping in the case of simulated time.
+     * This class is not useful when simulated time is not being used.
+     */
+    public static class SmartThread extends Thread {
+        public boolean isSleeping() {
+            return Time.isThreadWaiting(this);
+        }
+        public SmartThread(Runnable r) {
+            super(r);
+        }
+    }
+
+    /**
+     * Creates a thread that calls the given code repeatedly, sleeping for an
+     * interval of seconds equal to the return value of the previous call.
+     *
+     * The given afn may be a callable that returns the number of seconds to
+     * sleep, or it may be a Callable that returns another Callable that in turn
+     * returns the number of seconds to sleep. In the latter case isFactory.
+     *
+     * @param afn the code to call on each iteration
+     * @param isDaemon whether the new thread should be a daemon thread
+     * @param eh code to call when afn throws an exception
+     * @param priority the new thread's priority
+     * @param isFactory whether afn returns a callable instead of sleep seconds
+     * @param startImmediately whether to start the thread before returning
+     * @param threadName a suffix to be appended to the thread name
+     * @return the newly created thread
+     * @see java.lang.Thread
+     */
+    public static SmartThread asyncLoop(final Callable afn,
+            boolean isDaemon, final Thread.UncaughtExceptionHandler eh,
+            int priority, final boolean isFactory, boolean startImmediately,
+            String threadName) {
+        SmartThread thread = new SmartThread(new Runnable() {
+            public void run() {
+                Object s;
+                try {
+                    Callable fn = isFactory ? (Callable) afn.call() : afn;
+                    while ((s = fn.call()) instanceof Long) {
+                        Time.sleepSecs((Long) s);
+                    }
+                } catch (Throwable t) {
+                    if (Utils.exceptionCauseIsInstanceOf(
+                            InterruptedException.class, t)) {
+                        LOG.info("Async loop interrupted!");
+                        return;
+                    }
+                    LOG.error("Async loop died!", t);
+                    throw new RuntimeException(t);
+                }
+            }
+        });
+        if (eh != null) {
+            thread.setUncaughtExceptionHandler(eh);
+        } else {
+            thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+                public void uncaughtException(Thread t, Throwable e) {
+                    LOG.error("Async loop died!", e);
+                    Utils.exitProcess(1, "Async loop died!");
+                }
+            });
+        }
+        thread.setDaemon(isDaemon);
+        thread.setPriority(priority);
+        if (threadName != null && !threadName.isEmpty()) {
+            thread.setName(thread.getName() +"-"+ threadName);
+        }
+        if (startImmediately) {
+            thread.start();
+        }
+        return thread;
+    }
+
+    /**
+     * Convenience method used when only the function and name suffix are given.
+     * @param afn the code to call on each iteration
+     * @param threadName a suffix to be appended to the thread name
+     * @return the newly created thread
+     * @see java.lang.Thread
+     */
+    public static SmartThread asyncLoop(final Callable afn, String threadName, final Thread.UncaughtExceptionHandler eh) {
+        return asyncLoop(afn, false, eh, Thread.NORM_PRIORITY, false, true,
+                threadName);
+    }
+
+    /**
+     * Convenience method used when only the function is given.
+     * @param afn the code to call on each iteration
+     * @return the newly created thread
+     */
+    public static SmartThread asyncLoop(final Callable afn) {
+        return asyncLoop(afn, false, null, Thread.NORM_PRIORITY, false, true,
+                null);
+    }
+
+    public static <T> List<T> interleaveAll(List<List<T>> nodeList) {
+        if (nodeList != null && nodeList.size() > 0) {
+            List<T> first = new ArrayList<T>();
+            List<List<T>> rest = new ArrayList<List<T>>();
+            for (List<T> node : nodeList) {
+                if (node != null && node.size() > 0) {
+                  first.add(node.get(0));
+                  rest.add(node.subList(1, node.size()));
+                }
+            }
+            List<T> interleaveRest = interleaveAll(rest);
+            if (interleaveRest != null) {
+                first.addAll(interleaveRest);
+            }
+            return first;
+        }
+        return null;
+      }
+
+    public static long bitXor(Long a, Long b) {
+        return a ^ b;
+    }
+
+    public static List<String> getRepeat(List<String> list) {
+        List<String> rtn = new ArrayList<String>();
+        Set<String> idSet = new HashSet<String>();
+
+        for (String id : list) {
+            if (idSet.contains(id)) {
+                rtn.add(id);
+            } else {
+                idSet.add(id);
+            }
+        }
+
+        return rtn;
+    }
+
+    /**
+     * converts a clojure PersistentMap to java HashMap
+     */
+    public static Map<String, Object> convertClojureMapToJavaMap(Map map) {
+        Map<String, Object> ret = new HashMap<>(map.size());
+        for (Object obj : map.entrySet()) {
+            Map.Entry entry = (Map.Entry) obj;
+            Keyword keyword = (Keyword) entry.getKey();
+            String key = keyword.getName();
+            if (key.startsWith(":")) {
+                key = key.substring(1, key.length());
+            }
+            Object value = entry.getValue();
+            ret.put(key, value);
+        }
+
+        return ret;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/zookeeper/LeaderElectorImp.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/zookeeper/LeaderElectorImp.java b/storm-core/src/jvm/org/apache/storm/zookeeper/LeaderElectorImp.java
new file mode 100644
index 0000000..74816c2
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/zookeeper/LeaderElectorImp.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.zookeeper;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.leader.LeaderLatch;
+import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
+import org.apache.curator.framework.recipes.leader.Participant;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.nimbus.ILeaderElector;
+import org.apache.storm.nimbus.NimbusInfo;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class LeaderElectorImp implements ILeaderElector {
+    private static Logger LOG = LoggerFactory.getLogger(LeaderElectorImp.class);
+    private final Map conf;
+    private final List<String> servers;
+    private final CuratorFramework zk;
+    private final String leaderlockPath;
+    private final String id;
+    private final AtomicReference<LeaderLatch> leaderLatch;
+    private final AtomicReference<LeaderLatchListener> leaderLatchListener;
+    private final BlobStore blobStore;
+
+    public LeaderElectorImp(Map conf, List<String> servers, CuratorFramework zk, String leaderlockPath, String id, AtomicReference<LeaderLatch> leaderLatch,
+            AtomicReference<LeaderLatchListener> leaderLatchListener, BlobStore blobStore) {
+        this.conf = conf;
+        this.servers = servers;
+        this.zk = zk;
+        this.leaderlockPath = leaderlockPath;
+        this.id = id;
+        this.leaderLatch = leaderLatch;
+        this.leaderLatchListener = leaderLatchListener;
+        this.blobStore = blobStore;
+    }
+
+    @Override
+    public void prepare(Map conf) {
+        // no-op for zookeeper implementation
+    }
+
+    @Override
+    public void addToLeaderLockQueue() throws Exception {
+        // if this latch is already closed, we need to create new instance.
+        if (LeaderLatch.State.CLOSED.equals(leaderLatch.get().getState())) {
+            leaderLatch.set(new LeaderLatch(zk, leaderlockPath));
+            leaderLatchListener.set(Zookeeper.leaderLatchListenerImpl(conf, zk, blobStore, leaderLatch.get()));
+            LOG.info("LeaderLatch was in closed state. Resetted the leaderLatch and listeners.");
+        }
+        // Only if the latch is not already started we invoke start
+        if (LeaderLatch.State.LATENT.equals(leaderLatch.get().getState())) {
+            leaderLatch.get().addListener(leaderLatchListener.get());
+            leaderLatch.get().start();
+            LOG.info("Queued up for leader lock.");
+        } else {
+            LOG.info("Node already in queue for leader lock.");
+        }
+    }
+
+    @Override
+    // Only started latches can be closed.
+    public void removeFromLeaderLockQueue() throws Exception {
+        if (LeaderLatch.State.STARTED.equals(leaderLatch.get().getState())) {
+            leaderLatch.get().close();
+            LOG.info("Removed from leader lock queue.");
+        } else {
+            LOG.info("leader latch is not started so no removeFromLeaderLockQueue needed.");
+        }
+    }
+
+    @Override
+    public boolean isLeader() throws Exception {
+        return leaderLatch.get().hasLeadership();
+    }
+
+    @Override
+    public NimbusInfo getLeader() {
+        try {
+            return Zookeeper.toNimbusInfo(leaderLatch.get().getLeader());
+        } catch (Exception e) {
+            throw Utils.wrapInRuntime(e);
+        }
+    }
+
+    @Override
+    public List<NimbusInfo> getAllNimbuses() throws Exception {
+        List<NimbusInfo> nimbusInfos = new ArrayList<>();
+        Collection<Participant> participants = leaderLatch.get().getParticipants();
+        for (Participant participant : participants) {
+            nimbusInfos.add(Zookeeper.toNimbusInfo(participant));
+        }
+        return nimbusInfos;
+    }
+
+    @Override
+    public void close() {
+        LOG.info("closing zookeeper connection of leader elector.");
+        zk.close();
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java b/storm-core/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java
new file mode 100644
index 0000000..5247558
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/zookeeper/ZkEventTypes.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.zookeeper;
+
+import org.apache.zookeeper.Watcher;
+
+import java.util.HashMap;
+
+public class ZkEventTypes {
+
+    private static HashMap<Watcher.Event.EventType, String> map;
+
+    static {
+        map = new HashMap<Watcher.Event.EventType, String>();
+
+        map.put(Watcher.Event.EventType.None, ":none");
+        map.put(Watcher.Event.EventType.NodeCreated, ":node-created");
+        map.put(Watcher.Event.EventType.NodeDeleted, ":node-deleted");
+        map.put(Watcher.Event.EventType.NodeDataChanged, ":node-data-changed");
+        map.put(Watcher.Event.EventType.NodeChildrenChanged, ":node-children-changed");
+
+    }
+
+    public static String getTypeName(Watcher.Event.EventType type) {
+        return map.get(type);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java b/storm-core/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java
new file mode 100644
index 0000000..66dc231
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/zookeeper/ZkKeeperStates.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.zookeeper;
+
+import org.apache.zookeeper.Watcher;
+
+import java.util.HashMap;
+
+public class ZkKeeperStates {
+
+    private static HashMap<Watcher.Event.KeeperState, String> map;
+
+    static {
+        map = new HashMap<Watcher.Event.KeeperState, String>();
+
+        map.put(Watcher.Event.KeeperState.AuthFailed, ":auth-failed");
+        map.put(Watcher.Event.KeeperState.SyncConnected, ":connected");
+        map.put(Watcher.Event.KeeperState.Disconnected, ":disconnected");
+        map.put(Watcher.Event.KeeperState.Expired, ":expired");
+    }
+
+    public static String getStateName(Watcher.Event.KeeperState state) {
+        return map.get(state);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/zookeeper/Zookeeper.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/zookeeper/Zookeeper.java b/storm-core/src/jvm/org/apache/storm/zookeeper/Zookeeper.java
new file mode 100644
index 0000000..0580f41
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/zookeeper/Zookeeper.java
@@ -0,0 +1,463 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.zookeeper;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Sets;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.api.CuratorEvent;
+import org.apache.curator.framework.api.CuratorEventType;
+import org.apache.curator.framework.api.CuratorListener;
+import org.apache.curator.framework.recipes.leader.LeaderLatch;
+import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
+import org.apache.curator.framework.recipes.leader.Participant;
+import org.apache.curator.framework.state.ConnectionStateListener;
+import org.apache.storm.Config;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.blobstore.KeyFilter;
+import org.apache.storm.callback.DefaultWatcherCallBack;
+import org.apache.storm.callback.WatcherCallBack;
+import org.apache.storm.cluster.ClusterUtils;
+import org.apache.storm.cluster.IStateStorage;
+import org.apache.storm.cluster.VersionedData;
+import org.apache.storm.nimbus.ILeaderElector;
+import org.apache.storm.nimbus.NimbusInfo;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.Utils;
+import org.apache.storm.utils.ZookeeperAuthInfo;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.server.NIOServerCnxnFactory;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.BindException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
+
+
+public class Zookeeper {
+    private static Logger LOG = LoggerFactory.getLogger(Zookeeper.class);
+
+    // A singleton instance allows us to mock delegated static methods in our
+    // tests by subclassing.
+    private static final Zookeeper INSTANCE = new Zookeeper();
+    private static Zookeeper _instance = INSTANCE;
+
+    /**
+     * Provide an instance of this class for delegates to use.  To mock out
+     * delegated methods, provide an instance of a subclass that overrides the
+     * implementation of the delegated method.
+     *
+     * @param u a Zookeeper instance
+     */
+    public static void setInstance(Zookeeper u) {
+        _instance = u;
+    }
+
+    /**
+     * Resets the singleton instance to the default. This is helpful to reset
+     * the class to its original functionality when mocking is no longer
+     * desired.
+     */
+    public static void resetInstance() {
+        _instance = INSTANCE;
+    }
+
+    public  CuratorFramework mkClientImpl(Map conf, List<String> servers, Object port, String root) {
+        return mkClientImpl(conf, servers, port, root, new DefaultWatcherCallBack());
+    }
+
+    public  CuratorFramework mkClientImpl(Map conf, List<String> servers, Object port, Map authConf) {
+        return mkClientImpl(conf, servers, port, "", new DefaultWatcherCallBack(), authConf);
+    }
+
+    public  CuratorFramework mkClientImpl(Map conf, List<String> servers, Object port, String root, Map authConf) {
+        return mkClientImpl(conf, servers, port, root, new DefaultWatcherCallBack(), authConf);
+    }
+
+    public static CuratorFramework mkClient(Map conf, List<String> servers, Object port, String root, final WatcherCallBack watcher, Map authConf) {
+        return _instance.mkClientImpl(conf, servers, port, root, watcher, authConf);
+    }
+
+    public  CuratorFramework mkClientImpl(Map conf, List<String> servers, Object port, String root, final WatcherCallBack watcher, Map authConf) {
+        CuratorFramework fk;
+        if (authConf != null) {
+            fk = Utils.newCurator(conf, servers, port, root, new ZookeeperAuthInfo(authConf));
+        } else {
+            fk = Utils.newCurator(conf, servers, port, root);
+        }
+
+        fk.getCuratorListenable().addListener(new CuratorListener() {
+            @Override
+            public void eventReceived(CuratorFramework _fk, CuratorEvent e) throws Exception {
+                if (e.getType().equals(CuratorEventType.WATCHED)) {
+                    WatchedEvent event = e.getWatchedEvent();
+                    watcher.execute(event.getState(), event.getType(), event.getPath());
+                }
+            }
+        });
+        fk.start();
+        return fk;
+    }
+
+    /**
+     * connect ZK, register Watch/unhandle Watch
+     *
+     * @return
+     */
+    public  CuratorFramework mkClientImpl(Map conf, List<String> servers, Object port, String root, final WatcherCallBack watcher) {
+        return mkClientImpl(conf, servers, port, root, watcher, null);
+    }
+
+    public static String createNode(CuratorFramework zk, String path, byte[] data, org.apache.zookeeper.CreateMode mode, List<ACL> acls) {
+        String ret = null;
+        try {
+            String npath = normalizePath(path);
+            ret = zk.create().creatingParentsIfNeeded().withMode(mode).withACL(acls).forPath(npath, data);
+        } catch (Exception e) {
+            throw Utils.wrapInRuntime(e);
+        }
+        return ret;
+    }
+
+    public static String createNode(CuratorFramework zk, String path, byte[] data, List<ACL> acls){
+        return createNode(zk, path, data, org.apache.zookeeper.CreateMode.PERSISTENT, acls);
+    }
+
+    public static boolean existsNode(CuratorFramework zk, String path, boolean watch){
+        Stat stat = null;
+        try {
+            if (watch) {
+                stat = zk.checkExists().watched().forPath(normalizePath(path));
+            } else {
+                stat = zk.checkExists().forPath(normalizePath(path));
+            }
+        } catch (Exception e) {
+            throw Utils.wrapInRuntime(e);
+        }
+        return stat != null;
+    }
+
+    public static void deleteNode(CuratorFramework zk, String path){
+        try {
+            String npath = normalizePath(path);
+            if (existsNode(zk, npath, false)) {
+                zk.delete().deletingChildrenIfNeeded().forPath(normalizePath(path));
+            }
+        } catch (Exception e) {
+            if (Utils.exceptionCauseIsInstanceOf(KeeperException.NodeExistsException.class, e)) {
+                // do nothing
+                LOG.info("delete {} failed.", path, e);
+            } else {
+                throw Utils.wrapInRuntime(e);
+            }
+        }
+    }
+
+    public static void mkdirs(CuratorFramework zk, String path, List<ACL> acls) {
+        _instance.mkdirsImpl(zk, path, acls);
+    }
+
+    public void mkdirsImpl(CuratorFramework zk, String path, List<ACL> acls) {
+        String npath = normalizePath(path);
+        if (npath.equals("/")) {
+            return;
+        }
+        if (existsNode(zk, npath, false)) {
+            return;
+        }
+        byte[] byteArray = new byte[1];
+        byteArray[0] = (byte) 7;
+        try {
+            createNode(zk, npath, byteArray, org.apache.zookeeper.CreateMode.PERSISTENT, acls);
+        } catch (Exception e) {
+            if (Utils.exceptionCauseIsInstanceOf(KeeperException.NodeExistsException.class, e)) {
+                // this can happen when multiple clients doing mkdir at same time
+            }
+        }
+    }
+
+    public static void syncPath(CuratorFramework zk, String path){
+        try {
+            zk.sync().forPath(normalizePath(path));
+        } catch (Exception e) {
+            throw Utils.wrapInRuntime(e);
+        }
+    }
+
+    public static void addListener(CuratorFramework zk, ConnectionStateListener listener) {
+        zk.getConnectionStateListenable().addListener(listener);
+    }
+
+    public static byte[] getData(CuratorFramework zk, String path, boolean watch){
+        try {
+            String npath = normalizePath(path);
+            if (existsNode(zk, npath, watch)) {
+                if (watch) {
+                    return zk.getData().watched().forPath(npath);
+                } else {
+                    return zk.getData().forPath(npath);
+                }
+            }
+        } catch (Exception e) {
+            if (Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) {
+                // this is fine b/c we still have a watch from the successful exists call
+            } else {
+                throw Utils.wrapInRuntime(e);
+            }
+        }
+        return null;
+    }
+
+    public static Integer getVersion(CuratorFramework zk, String path, boolean watch) throws Exception {
+        String npath = normalizePath(path);
+        Stat stat = null;
+        if (existsNode(zk, npath, watch)) {
+            if (watch) {
+                stat = zk.checkExists().watched().forPath(npath);
+            } else {
+                stat = zk.checkExists().forPath(npath);
+            }
+        }
+        return stat == null ? null : Integer.valueOf(stat.getVersion());
+    }
+
+    public static List<String> getChildren(CuratorFramework zk, String path, boolean watch) {
+        try {
+            String npath = normalizePath(path);
+            if (watch) {
+                return zk.getChildren().watched().forPath(npath);
+            } else {
+                return zk.getChildren().forPath(npath);
+            }
+        } catch (Exception e) {
+            throw Utils.wrapInRuntime(e);
+        }
+    }
+
+    // Deletes the state inside the zookeeper for a key, for which the
+    // contents of the key starts with nimbus host port information
+    public static void deleteNodeBlobstore(CuratorFramework zk, String parentPath, String hostPortInfo){
+        String normalizedPatentPath = normalizePath(parentPath);
+        List<String> childPathList = null;
+        if (existsNode(zk, normalizedPatentPath, false)) {
+            childPathList = getChildren(zk, normalizedPatentPath, false);
+            for (String child : childPathList) {
+                if (child.startsWith(hostPortInfo)) {
+                    LOG.debug("deleteNode child {}", child);
+                    deleteNode(zk, normalizedPatentPath + "/" + child);
+                }
+            }
+        }
+    }
+
+    public static Stat setData(CuratorFramework zk, String path, byte[] data){
+        try {
+            String npath = normalizePath(path);
+            return zk.setData().forPath(npath, data);
+        } catch (Exception e) {
+            throw Utils.wrapInRuntime(e);
+        }
+    }
+
+    public static boolean exists(CuratorFramework zk, String path, boolean watch){
+        return existsNode(zk, path, watch);
+    }
+
+    public static List mkInprocessZookeeper(String localdir, Integer port) throws Exception {
+        File localfile = new File(localdir);
+        ZooKeeperServer zk = new ZooKeeperServer(localfile, localfile, 2000);
+        NIOServerCnxnFactory factory = null;
+        int report = 2000;
+        int limitPort = 65535;
+        if (port != null) {
+            report = port;
+            limitPort = port;
+        }
+        while (true) {
+            try {
+                factory = new NIOServerCnxnFactory();
+                factory.configure(new InetSocketAddress(report), 0);
+                break;
+            } catch (BindException e) {
+                report++;
+                if (report > limitPort) {
+                    throw new RuntimeException("No port is available to launch an inprocess zookeeper");
+                }
+            }
+        }
+        LOG.info("Starting inprocess zookeeper at port {} and dir {}", report, localdir);
+        factory.startup(zk);
+        return Arrays.asList((Object) new Long(report), (Object) factory);
+    }
+
+    public static void shutdownInprocessZookeeper(NIOServerCnxnFactory handle) {
+        handle.shutdown();
+    }
+
+    public static NimbusInfo toNimbusInfo(Participant participant) {
+        String id = participant.getId();
+        if (StringUtils.isBlank(id)) {
+            throw new RuntimeException("No nimbus leader participant host found, have you started your nimbus hosts?");
+        }
+        NimbusInfo nimbusInfo = NimbusInfo.parse(id);
+        nimbusInfo.setLeader(participant.isLeader());
+        return nimbusInfo;
+    }
+
+    // Leader latch listener that will be invoked when we either gain or lose leadership
+    public static LeaderLatchListener leaderLatchListenerImpl(final Map conf, final CuratorFramework zk, final BlobStore blobStore, final LeaderLatch leaderLatch) throws UnknownHostException {
+        final String hostName = InetAddress.getLocalHost().getCanonicalHostName();
+        return new LeaderLatchListener() {
+            @Override
+            public void isLeader() {
+                Set<String> activeTopologyIds = new HashSet<>(Zookeeper.getChildren(zk, conf.get(Config.STORM_ZOOKEEPER_ROOT) + ClusterUtils.STORMS_SUBTREE, false));
+                Set<String> localTopologyIds = blobStore.filterAndListKeys(new KeyFilter<String>() {
+                    @Override
+                    public String filter(String key) {
+                        return ConfigUtils.getIdFromBlobKey(key);
+                    }
+                });
+                Sets.SetView<String> diffTopology = Sets.difference(activeTopologyIds, localTopologyIds);
+                LOG.info("active-topology-ids [{}] local-topology-ids [{}] diff-topology [{}]",
+                        generateJoinedString(activeTopologyIds), generateJoinedString(localTopologyIds),
+                        generateJoinedString(diffTopology));
+
+                if (diffTopology.isEmpty()) {
+                    LOG.info("Accepting leadership, all active topology found locally.");
+                } else {
+                    LOG.info("code for all active topologies not available locally, giving up leadership.");
+                    try {
+                        leaderLatch.close();
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                }
+            }
+
+            @Override
+            public void notLeader() {
+                LOG.info("{} lost leadership.", hostName);
+            }
+
+            private String generateJoinedString(Set<String> activeTopologyIds) {
+                return Joiner.on(",").join(activeTopologyIds);
+            }
+        };
+    }
+
+    public static ILeaderElector zkLeaderElector(Map conf, BlobStore blobStore) throws UnknownHostException {
+        return _instance.zkLeaderElectorImpl(conf, blobStore);
+    }
+
+    protected ILeaderElector zkLeaderElectorImpl(Map conf, BlobStore blobStore) throws UnknownHostException {
+        List<String> servers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS);
+        Object port = conf.get(Config.STORM_ZOOKEEPER_PORT);
+        CuratorFramework zk = mkClientImpl(conf, servers, port, "", conf);
+        String leaderLockPath = conf.get(Config.STORM_ZOOKEEPER_ROOT) + "/leader-lock";
+        String id = NimbusInfo.fromConf(conf).toHostPortString();
+        AtomicReference<LeaderLatch> leaderLatchAtomicReference = new AtomicReference<>(new LeaderLatch(zk, leaderLockPath, id));
+        AtomicReference<LeaderLatchListener> leaderLatchListenerAtomicReference =
+                new AtomicReference<>(leaderLatchListenerImpl(conf, zk, blobStore, leaderLatchAtomicReference.get()));
+        return new LeaderElectorImp(conf, servers, zk, leaderLockPath, id, leaderLatchAtomicReference,
+            leaderLatchListenerAtomicReference, blobStore);
+    }
+
+    /**
+     * Get the data along with a version
+     * @param zk the zk instance to use
+     * @param path the path to get it from
+     * @param watch should a watch be enabled
+     * @return null if no data is found, else the data with the version.
+     */
+    public static VersionedData<byte[]> getDataWithVersion(CuratorFramework zk, String path, boolean watch) {
+        VersionedData<byte[]> data = null;
+        try {
+            byte[] bytes = null;
+            Stat stats = new Stat();
+            String npath = normalizePath(path);
+            if (existsNode(zk, npath, watch)) {
+                if (watch) {
+                    bytes = zk.getData().storingStatIn(stats).watched().forPath(npath);
+                } else {
+                    bytes = zk.getData().storingStatIn(stats).forPath(npath);
+                }
+                if (bytes != null) {
+                    int version = stats.getVersion();
+                    data = new VersionedData<>(version, bytes);
+                }
+            }
+        } catch (Exception e) {
+            if (Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) {
+                // this is fine b/c we still have a watch from the successful exists call
+            } else {
+                Utils.wrapInRuntime(e);
+            }
+        }
+        return data;
+    }
+
+    public static List<String> tokenizePath(String path) {
+        String[] toks = path.split("/");
+        java.util.ArrayList<String> rtn = new ArrayList<String>();
+        for (String str : toks) {
+            if (!str.isEmpty()) {
+                rtn.add(str);
+            }
+        }
+        return rtn;
+    }
+
+    public static String parentPath(String path) {
+        List<String> toks = Zookeeper.tokenizePath(path);
+        int size = toks.size();
+        if (size > 0) {
+            toks.remove(size - 1);
+        }
+        return Zookeeper.toksToPath(toks);
+    }
+
+    public static String toksToPath(List<String> toks) {
+        StringBuffer buff = new StringBuffer();
+        buff.append("/");
+        int size = toks.size();
+        for (int i = 0; i < size; i++) {
+            buff.append(toks.get(i));
+            if (i < (size - 1)) {
+                buff.append("/");
+            }
+        }
+        return buff.toString();
+    }
+
+    public static String normalizePath(String path) {
+        String rtn = toksToPath(tokenizePath(path));
+        return rtn;
+    }
+}


[20/21] storm git commit: Merge branch 'STORM-2018-1.x' of https://github.com/revans2/incubator-storm into STORM-2018-1.x-test

Posted by ka...@apache.org.
Merge branch 'STORM-2018-1.x' of https://github.com/revans2/incubator-storm into STORM-2018-1.x-test


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0e0cae4a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0e0cae4a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0e0cae4a

Branch: refs/heads/1.x-branch
Commit: 0e0cae4aff656768e7ca11871856a91db646f71e
Parents: e59447b df33efd
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Oct 28 00:02:45 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Oct 28 00:02:45 2016 +0900

----------------------------------------------------------------------
 bin/storm.py                                    |    6 +-
 log4j2/cluster.xml                              |    2 +-
 .../cluster_state/zookeeper_state_factory.clj   |    5 +-
 .../org/apache/storm/command/healthcheck.clj    |   88 --
 .../org/apache/storm/command/kill_workers.clj   |   33 -
 .../clj/org/apache/storm/daemon/executor.clj    |    4 +-
 .../apache/storm/daemon/local_supervisor.clj    |   28 +
 .../clj/org/apache/storm/daemon/logviewer.clj   |    7 +-
 .../clj/org/apache/storm/daemon/supervisor.clj  | 1272 ------------------
 .../src/clj/org/apache/storm/daemon/worker.clj  |    1 +
 storm-core/src/clj/org/apache/storm/event.clj   |   71 -
 .../storm/pacemaker/pacemaker_state_factory.clj |    6 +-
 storm-core/src/clj/org/apache/storm/testing.clj |   74 +-
 .../jvm/org/apache/storm/ProcessSimulator.java  |   82 ++
 .../src/jvm/org/apache/storm/StormTimer.java    |  242 ++++
 .../storm/callback/DefaultWatcherCallBack.java  |   35 +
 .../apache/storm/callback/WatcherCallBack.java  |   24 +
 .../storm/callback/ZKStateChangedCallback.java  |   25 +
 .../storm/cluster/ClusterStateFactory.java      |    4 +-
 .../org/apache/storm/cluster/ClusterUtils.java  |  244 ++++
 .../org/apache/storm/cluster/ExecutorBeat.java  |   44 +
 .../org/apache/storm/cluster/IStateStorage.java |  219 +++
 .../storm/cluster/IStormClusterState.java       |  126 ++
 .../storm/cluster/StateStorageFactory.java      |   28 +
 .../storm/cluster/StormClusterStateImpl.java    |  720 ++++++++++
 .../org/apache/storm/cluster/VersionedData.java |   36 +
 .../apache/storm/cluster/ZKStateStorage.java    |  252 ++++
 .../org/apache/storm/command/HealthCheck.java   |  126 ++
 .../org/apache/storm/command/KillWorkers.java   |   38 +
 .../org/apache/storm/daemon/DaemonCommon.java   |   22 +
 .../storm/daemon/metrics/MetricsUtils.java      |   19 +-
 .../storm/daemon/supervisor/AdvancedFSOps.java  |  350 +++++
 .../storm/daemon/supervisor/BasicContainer.java |  713 ++++++++++
 .../supervisor/BasicContainerLauncher.java      |   59 +
 .../storm/daemon/supervisor/Container.java      |  534 ++++++++
 .../daemon/supervisor/ContainerLauncher.java    |   96 ++
 .../supervisor/ContainerRecoveryException.java  |   29 +
 .../DefaultUncaughtExceptionHandler.java        |   31 +
 .../supervisor/EventManagerPushCallback.java    |   37 +
 .../daemon/supervisor/ExitCodeCallback.java     |   30 +
 .../storm/daemon/supervisor/Killable.java       |   50 +
 .../storm/daemon/supervisor/LocalContainer.java |   85 ++
 .../supervisor/LocalContainerLauncher.java      |   60 +
 .../daemon/supervisor/ReadClusterState.java     |  327 +++++
 .../daemon/supervisor/RunAsUserContainer.java   |   96 ++
 .../supervisor/RunAsUserContainerLauncher.java  |   57 +
 .../apache/storm/daemon/supervisor/Slot.java    |  785 +++++++++++
 .../daemon/supervisor/StandaloneSupervisor.java |   83 ++
 .../storm/daemon/supervisor/Supervisor.java     |  362 +++++
 .../daemon/supervisor/SupervisorUtils.java      |  321 +++++
 .../supervisor/timer/SupervisorHealthCheck.java |   41 +
 .../supervisor/timer/SupervisorHeartbeat.java   |   87 ++
 .../daemon/supervisor/timer/UpdateBlobs.java    |  105 ++
 .../org/apache/storm/event/EventManager.java    |   24 +
 .../org/apache/storm/event/EventManagerImp.java |   97 ++
 .../apache/storm/localizer/AsyncLocalizer.java  |  432 ++++++
 .../org/apache/storm/localizer/ILocalizer.java  |   70 +
 .../localizer/LocalDownloadedResource.java      |  146 ++
 .../LocalizedResourceRetentionSet.java          |    2 +-
 .../storm/localizer/LocalizedResourceSet.java   |    8 +-
 .../org/apache/storm/localizer/Localizer.java   |   38 +-
 .../storm/metric/StormMetricsRegistry.java      |   88 ++
 .../org/apache/storm/nimbus/ILeaderElector.java |    6 +-
 .../serialization/SerializationFactory.java     |   17 +-
 .../org/apache/storm/testing/FeederSpout.java   |   27 +-
 .../staticmocking/MockedSupervisorUtils.java    |   31 +
 .../apache/storm/trident/util/TridentUtils.java |    2 +-
 .../jvm/org/apache/storm/utils/ConfigUtils.java |  536 ++++++++
 .../org/apache/storm/utils/InprocMessaging.java |   69 +-
 .../jvm/org/apache/storm/utils/LocalState.java  |  112 +-
 .../org/apache/storm/utils/NimbusClient.java    |   19 +-
 .../src/jvm/org/apache/storm/utils/Time.java    |   51 +-
 .../src/jvm/org/apache/storm/utils/Utils.java   | 1054 +++++++++++++--
 .../storm/zookeeper/LeaderElectorImp.java       |  124 ++
 .../apache/storm/zookeeper/ZkEventTypes.java    |   43 +
 .../apache/storm/zookeeper/ZkKeeperStates.java  |   41 +
 .../org/apache/storm/zookeeper/Zookeeper.java   |  463 +++++++
 .../src/native/worker-launcher/impl/main.c      |   20 +-
 .../worker-launcher/impl/worker-launcher.c      |   46 +-
 .../worker-launcher/impl/worker-launcher.h      |    2 +-
 .../org/apache/storm/integration_test.clj       |   97 +-
 .../org/apache/storm/testing4j_test.clj         |   11 +-
 .../clj/org/apache/storm/logviewer_test.clj     |   55 +-
 .../test/clj/org/apache/storm/metrics_test.clj  |    9 +-
 .../test/clj/org/apache/storm/nimbus_test.clj   |   10 +-
 .../storm/security/auth/drpc_auth_test.clj      |   32 +-
 .../clj/org/apache/storm/supervisor_test.clj    |  751 -----------
 .../daemon/supervisor/BasicContainerTest.java   |  490 +++++++
 .../storm/daemon/supervisor/ContainerTest.java  |  265 ++++
 .../storm/daemon/supervisor/SlotTest.java       |  515 +++++++
 .../storm/localizer/AsyncLocalizerTest.java     |  190 +++
 .../LocalizedResourceRetentionSetTest.java      |   10 +-
 .../localizer/LocalizedResourceSetTest.java     |   12 +-
 .../apache/storm/localizer/LocalizerTest.java   |    9 +-
 94 files changed, 11618 insertions(+), 2627 deletions(-)
----------------------------------------------------------------------



[17/21] storm git commit: Merge branch '1.x-branch' into STORM-2018-1.x

Posted by ka...@apache.org.
Merge branch '1.x-branch' into STORM-2018-1.x

Conflicts:
	storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1d4d414d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1d4d414d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1d4d414d

Branch: refs/heads/1.x-branch
Commit: 1d4d414d9707a2ab4c29b8bba06ea20eef6950a5
Parents: f12b7da a11e470
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Tue Oct 25 14:06:43 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Tue Oct 25 14:06:43 2016 -0500

----------------------------------------------------------------------
 .gitignore                                      |    3 +
 .travis.yml                                     |    9 +-
 CHANGELOG.md                                    |   29 +
 bin/storm.cmd                                   |    4 +-
 bin/storm.py                                    |   10 +-
 conf/defaults.yaml                              |    2 +-
 dev-tools/travis/travis-script.sh               |   36 +-
 docs/Guaranteeing-message-processing.md         |    4 +-
 docs/Pacemaker.md                               |   18 +-
 docs/Resource_Aware_Scheduler_overview.md       |  253 +++-
 docs/STORM-UI-REST-API.md                       |  111 +-
 docs/Trident-API-Overview.md                    |   17 +
 docs/images/Storm-JMS-Example.png               |  Bin 0 -> 40432 bytes
 .../ras_new_strategy_network_cdf_random.png     |  Bin 0 -> 157859 bytes
 ...tegy_network_metric_cdf_yahoo_topologies.png |  Bin 0 -> 152034 bytes
 ...rategy_network_metric_improvement_random.png |  Bin 0 -> 68976 bytes
 .../ras_new_strategy_network_metric_random.png  |  Bin 0 -> 79004 bytes
 ...strategy_network_metric_yahoo_topologies.png |  Bin 0 -> 75195 bytes
 docs/images/ras_new_strategy_runtime_random.png |  Bin 0 -> 76534 bytes
 docs/images/ras_new_strategy_runtime_yahoo.png  |  Bin 0 -> 71727 bytes
 docs/index.md                                   |    1 +
 docs/storm-jms-example.md                       |  111 ++
 docs/storm-jms-spring.md                        |   25 +
 docs/storm-jms.md                               |   33 +
 docs/storm-sql-example.md                       |  372 +++++
 docs/storm-sql-internal.md                      |   10 +-
 docs/storm-sql-reference.md                     | 1206 +++++++++++++++
 docs/storm-sql.md                               |   54 +-
 .../starter/spout/RandomSentenceSpout.java      |   39 +-
 .../starter/trident/DebugMemoryMapState.java    |   73 +
 .../starter/trident/TridentKafkaWordCount.java  |   58 +-
 .../starter/trident/TridentMapExample.java      |   11 +-
 .../java/org/apache/storm/flux/TCKTest.java     |   10 +
 .../test/resources/configs/bad_shell_test.yaml  |  118 ++
 .../src/test/resources/configs/shell_test.yaml  |   14 +
 .../flux/wrappers/bolts/FluxShellBolt.java      |   58 +-
 .../flux/wrappers/spouts/FluxShellSpout.java    |   55 +-
 external/flux/pom.xml                           |   17 -
 external/sql/README.md                          |   46 +
 external/sql/pom.xml                            |    1 +
 external/sql/storm-sql-core/pom.xml             |   35 +-
 .../storm-sql-core/src/codegen/data/Parser.tdd  |    6 +-
 .../src/jvm/org/apache/storm/sql/StormSql.java  |    5 +
 .../jvm/org/apache/storm/sql/StormSqlImpl.java  |   46 +-
 .../org/apache/storm/sql/StormSqlRunner.java    |   55 +-
 .../apache/storm/sql/compiler/ExprCompiler.java |  523 -------
 .../RexNodeToBlockStatementCompiler.java        |  102 ++
 .../backends/standalone/PlanCompiler.java       |   17 +-
 .../backends/standalone/RelNodeCompiler.java    |  136 +-
 .../compiler/backends/trident/PlanCompiler.java |    9 +-
 .../trident/TridentLogicalPlanCompiler.java     |   93 +-
 .../test/org/apache/storm/sql/TestStormSql.java |   77 +-
 .../storm/sql/compiler/TestCompilerUtils.java   |   34 +-
 .../storm/sql/compiler/TestExprCompiler.java    |   93 --
 .../storm/sql/compiler/TestExprSemantic.java    |  262 +++-
 .../backends/standalone/TestPlanCompiler.java   |    6 +-
 .../standalone/TestRelNodeCompiler.java         |    2 +-
 .../backends/trident/TestPlanCompiler.java      |  211 ++-
 .../org/apache/storm/sql/kafka/JsonScheme.java  |   58 -
 .../apache/storm/sql/kafka/JsonSerializer.java  |   56 -
 .../sql/kafka/KafkaDataSourcesProvider.java     |   61 +-
 .../storm/sql/kafka/TestJsonRepresentation.java |   50 -
 .../sql/kafka/TestKafkaDataSourcesProvider.java |   26 +-
 .../storm-sql-external/storm-sql-redis/pom.xml  |   80 +
 .../sql/redis/RedisDataSourcesProvider.java     |  246 ++++
 ...apache.storm.sql.runtime.DataSourcesProvider |   16 +
 .../sql/redis/TestRedisDataSourcesProvider.java |  162 ++
 external/sql/storm-sql-runtime/pom.xml          |   13 +-
 .../calcite/interpreter/StormContext.java       |   31 +
 .../sql/runtime/AbstractChannelHandler.java     |   32 +-
 .../sql/runtime/AbstractValuesProcessor.java    |   33 +-
 .../storm/sql/runtime/ChannelContext.java       |   32 +-
 .../storm/sql/runtime/ChannelHandler.java       |   32 +-
 .../org/apache/storm/sql/runtime/Channels.java  |   32 +-
 .../apache/storm/sql/runtime/DataSource.java    |   32 +-
 .../storm/sql/runtime/DataSourcesProvider.java  |   33 +-
 .../storm/sql/runtime/DataSourcesRegistry.java  |   33 +-
 .../org/apache/storm/sql/runtime/FieldInfo.java |    4 +-
 .../sql/runtime/ISqlTridentDataSource.java      |   41 +-
 .../sql/runtime/SimpleSqlTridentConsumer.java   |   41 +
 .../storm/sql/runtime/StormSqlFunctions.java    |   32 +-
 .../sql/runtime/calcite/StormDataContext.java   |   79 +
 .../sql/runtime/serde/json/JsonScheme.java      |   58 +
 .../sql/runtime/serde/json/JsonSerializer.java  |   57 +
 .../trident/AbstractTridentProcessor.java       |   33 +-
 .../trident/functions/EvaluationFilter.java     |   21 +-
 .../trident/functions/EvaluationFunction.java   |   22 +-
 .../storm/sql/TestJsonRepresentation.java       |   52 +
 .../test/org/apache/storm/sql/TestUtils.java    |  241 ++-
 external/storm-cassandra/pom.xml                |   15 -
 external/storm-eventhubs/pom.xml                |   21 +-
 .../storm/eventhubs/bolt/EventHubBolt.java      |   29 +-
 .../eventhubs/spout/BinaryEventDataScheme.java  |   66 +
 .../storm/eventhubs/spout/EventDataScheme.java  |   38 +-
 .../storm/eventhubs/spout/EventHubSpout.java    |    8 +-
 .../eventhubs/spout/EventHubSpoutConfig.java    |  119 +-
 .../storm/eventhubs/spout/FieldConstants.java   |    1 +
 .../storm/eventhubs/spout/IEventDataScheme.java |   15 +-
 .../eventhubs/spout/StringEventDataScheme.java  |   69 +
 external/storm-hdfs/pom.xml                     |    9 -
 external/storm-jms/README.markdown              |   25 +
 external/storm-jms/core/pom.xml                 |   95 ++
 .../apache/storm/jms/JmsMessageProducer.java    |   46 +
 .../java/org/apache/storm/jms/JmsProvider.java  |   48 +
 .../org/apache/storm/jms/JmsTupleProducer.java  |   58 +
 .../java/org/apache/storm/jms/bolt/JmsBolt.java |  217 +++
 .../apache/storm/jms/spout/JmsMessageID.java    |   58 +
 .../org/apache/storm/jms/spout/JmsSpout.java    |  382 +++++
 .../org/apache/storm/jms/trident/JmsBatch.java  |   27 +
 .../org/apache/storm/jms/trident/JmsState.java  |  129 ++
 .../storm/jms/trident/JmsStateFactory.java      |   40 +
 .../apache/storm/jms/trident/JmsUpdater.java    |   38 +
 .../storm/jms/trident/TridentJmsSpout.java      |  409 +++++
 .../apache/storm/jms/spout/JmsSpoutTest.java    |   88 ++
 .../apache/storm/jms/spout/MockJmsProvider.java |   62 +
 .../jms/spout/MockSpoutOutputCollector.java     |   55 +
 .../storm/jms/spout/MockTupleProducer.java      |   47 +
 .../core/src/test/resources/jndi.properties     |   18 +
 external/storm-jms/examples/README.markdown     |   12 +
 external/storm-jms/examples/pom.xml             |  151 ++
 .../storm/jms/example/ExampleJmsTopology.java   |  131 ++
 .../apache/storm/jms/example/GenericBolt.java   |  116 ++
 .../storm/jms/example/JsonTupleProducer.java    |   58 +
 .../storm/jms/example/SpringJmsProvider.java    |   74 +
 .../src/main/resources/jms-activemq.xml         |   53 +
 .../src/main/resources/log4j.properties         |   29 +
 external/storm-jms/pom.xml                      |   70 +
 .../jvm/org/apache/storm/kafka/KafkaUtils.java  |    9 +-
 .../apache/storm/kafka/PartitionManager.java    |    2 +-
 .../org/apache/storm/kafka/bolt/KafkaBolt.java  |    6 +-
 .../trident/TransactionalTridentKafkaSpout.java |    8 +-
 .../kafka/trident/TridentKafkaEmitter.java      |   30 +-
 .../storm/redis/bolt/AbstractRedisBolt.java     |    5 +
 .../common/container/JedisClusterContainer.java |    3 +-
 .../JedisCommandsInstanceContainer.java         |   10 +-
 .../redis/common/container/JedisContainer.java  |    3 +-
 integration-test/README.md                      |   59 +
 integration-test/config/Vagrantfile             |  183 +++
 integration-test/config/cluster.xml             |   96 ++
 integration-test/config/common.sh               |   21 +
 integration-test/config/etc-hosts               |   18 +
 integration-test/config/install-storm.sh        |   39 +
 integration-test/config/install-zookeeper.sh    |   20 +
 integration-test/config/storm.yaml              |   33 +
 integration-test/pom.xml                        |  250 ++++
 integration-test/run-it.sh                      |   83 ++
 .../org/apache/storm/ExclamationTopology.java   |   93 ++
 .../org/apache/storm/debug/DebugHelper.java     |   39 +
 .../storm/st/topology/TestableTopology.java     |   30 +
 .../topology/window/SlidingTimeCorrectness.java |  170 +++
 .../window/SlidingWindowCorrectness.java        |  157 ++
 .../window/TumblingTimeCorrectness.java         |  167 +++
 .../window/TumblingWindowCorrectness.java       |  154 ++
 .../storm/st/topology/window/data/FromJson.java |   22 +
 .../storm/st/topology/window/data/TimeData.java |  110 ++
 .../st/topology/window/data/TimeDataWindow.java |   91 ++
 .../apache/storm/st/utils/StringDecorator.java  |   37 +
 .../org/apache/storm/st/utils/TimeUtil.java     |   54 +
 .../test/java/org/apache/storm/st/DemoTest.java |   85 ++
 .../apache/storm/st/helper/AbstractTest.java    |   27 +
 .../apache/storm/st/meta/TestngListener.java    |   97 ++
 .../st/tests/window/SlidingWindowTest.java      |  187 +++
 .../st/tests/window/TumblingWindowTest.java     |   99 ++
 .../org/apache/storm/st/utils/AssertUtil.java   |   71 +
 .../org/apache/storm/st/wrapper/LogData.java    |   66 +
 .../apache/storm/st/wrapper/StormCluster.java   |  118 ++
 .../org/apache/storm/st/wrapper/TopoWrap.java   |  378 +++++
 .../src/test/resources/storm-conf/storm.yaml    |   33 +
 pom.xml                                         |   28 +-
 storm-core/pom.xml                              |   16 +
 .../src/clj/org/apache/storm/daemon/nimbus.clj  |  118 +-
 .../src/clj/org/apache/storm/daemon/worker.clj  |    5 +-
 .../storm/pacemaker/pacemaker_state_factory.clj |  253 +++-
 .../src/clj/org/apache/storm/testing4j.clj      |    6 +-
 storm-core/src/clj/org/apache/storm/ui/core.clj |   35 +-
 storm-core/src/jvm/org/apache/storm/Config.java |   10 +-
 .../org/apache/storm/generated/Assignment.java  |  244 +--
 .../storm/generated/ClusterWorkerHeartbeat.java |   52 +-
 .../storm/generated/CommonAggregateStats.java   |  166 ++-
 .../storm/generated/ComponentPageInfo.java      |  386 +++--
 .../org/apache/storm/generated/Credentials.java |   44 +-
 .../jvm/org/apache/storm/generated/HBNodes.java |   32 +-
 .../org/apache/storm/generated/HBRecords.java   |   36 +-
 .../storm/generated/LSApprovedWorkers.java      |   44 +-
 .../generated/LSSupervisorAssignments.java      |   48 +-
 .../apache/storm/generated/LSTopoHistory.java   |   64 +-
 .../storm/generated/LSTopoHistoryList.java      |   36 +-
 .../storm/generated/LSWorkerHeartbeat.java      |   36 +-
 .../apache/storm/generated/ListBlobsResult.java |   32 +-
 .../apache/storm/generated/LocalAssignment.java |   36 +-
 .../apache/storm/generated/LocalStateData.java  |   48 +-
 .../org/apache/storm/generated/LogConfig.java   |   48 +-
 .../jvm/org/apache/storm/generated/Nimbus.java  |   36 +-
 .../org/apache/storm/generated/NodeInfo.java    |   32 +-
 .../storm/generated/RebalanceOptions.java       |   44 +-
 .../storm/generated/SettableBlobMeta.java       |   36 +-
 .../org/apache/storm/generated/StormBase.java   |   92 +-
 .../apache/storm/generated/SupervisorInfo.java  |  152 +-
 .../storm/generated/SupervisorPageInfo.java     |   72 +-
 .../storm/generated/TopologyHistoryInfo.java    |   32 +-
 .../storm/generated/TopologyPageInfo.java       |  132 +-
 .../apache/storm/generated/TopologyStats.java   |  220 +--
 .../apache/storm/generated/WorkerSummary.java   |   44 +-
 .../apache/storm/messaging/netty/Client.java    |    1 -
 .../netty/KerberosSaslClientHandler.java        |    6 +-
 .../netty/KerberosSaslNettyClient.java          |    5 +-
 .../jvm/org/apache/storm/nimbus/NimbusInfo.java |    2 +
 .../apache/storm/pacemaker/PacemakerClient.java |   72 +-
 .../pacemaker/PacemakerConnectionException.java |   24 +
 .../pacemaker/codec/ThriftNettyClientCodec.java |    7 +-
 .../scheduler/multitenant/IsolatedPool.java     |   10 +-
 .../storm/scheduler/resource/RAS_Node.java      |   12 +
 .../storm/scheduler/resource/ResourceUtils.java |   22 +
 .../DefaultResourceAwareStrategy.java           |  673 +++++----
 .../apache/storm/testing/MkClusterParam.java    |   14 +-
 .../TransactionalSpoutCoordinator.java          |    7 +-
 .../jvm/org/apache/storm/trident/Stream.java    |   40 +
 .../OperationAwareFlatMapFunction.java          |   24 +
 .../operation/OperationAwareMapFunction.java    |   24 +
 .../storm/trident/operation/builtin/Debug.java  |   18 +-
 .../operation/impl/FlatMapFunctionExecutor.java |   21 +
 .../operation/impl/MapFunctionExecutor.java     |   21 +
 .../spout/IOpaquePartitionedTridentSpout.java   |   11 +-
 .../trident/spout/IPartitionedTridentSpout.java |   10 +-
 .../OpaquePartitionedTridentSpoutExecutor.java  |   62 +-
 .../spout/PartitionedTridentSpoutExecutor.java  |   24 +-
 .../trident/state/CombinerValueUpdater.java     |    9 +
 .../topology/MasterBatchCoordinator.java        |   39 +-
 .../state/RotatingTransactionalState.java       |   59 +-
 .../topology/state/TransactionalState.java      |   44 +-
 .../org/apache/storm/utils/NimbusClient.java    |    4 +-
 .../storm/utils/ShellBoltMessageQueue.java      |    2 +-
 storm-core/src/py/storm/Nimbus.py               |   14 +-
 storm-core/src/py/storm/ttypes.py               |  756 +++++-----
 storm-core/src/storm.thrift                     |    2 +
 storm-core/src/ui/public/index.html             |  126 +-
 storm-core/src/ui/public/js/script.js           |   34 +-
 .../templates/component-page-template.html      |   22 +
 .../public/templates/index-page-template.html   |  117 +-
 .../templates/supervisor-page-template.html     |   16 +
 .../templates/topology-page-template.html       |  102 +-
 storm-core/src/ui/public/topology.html          |    6 +-
 .../org/apache/storm/integration_test.clj       |    3 +-
 .../storm/pacemaker_state_factory_test.clj      |   67 +-
 .../resource/TestResourceAwareScheduler.java    | 1394 +++++++++---------
 .../TestUtilsForResourceAwareScheduler.java     |   74 +-
 .../eviction/TestDefaultEvictionStrategy.java   |  751 ++++++++++
 .../TestDefaultResourceAwareStrategy.java       |    8 +-
 storm-dist/binary/src/main/assembly/binary.xml  |   28 +
 249 files changed, 15587 insertions(+), 4263 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/1d4d414d/bin/storm.py
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/1d4d414d/storm-core/src/clj/org/apache/storm/daemon/worker.clj
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/1d4d414d/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
index 077d528,b37b901..2bbeda4
--- a/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
+++ b/storm-core/src/clj/org/apache/storm/pacemaker/pacemaker_state_factory.clj
@@@ -24,9 -25,10 +25,10 @@@
               [util :as util]])
    (:import [org.apache.storm.generated
              HBExecutionException HBServerMessageType HBMessage
-             HBMessageData HBPulse]
+             HBMessageData HBPulse ClusterWorkerHeartbeat]
             [org.apache.storm.cluster_state zookeeper_state_factory]
 -           [org.apache.storm.cluster ClusterState]
 +           [org.apache.storm.cluster ClusterState ZKStateStorage]
+            [org.apache.storm.utils Utils]
             [org.apache.storm.pacemaker PacemakerClient])
    (:gen-class
     :implements [org.apache.storm.cluster.ClusterStateFactory]))
@@@ -38,15 -53,108 +53,112 @@@
  (defn makeZKState [conf auth-conf acls context]
    (.mkState (zookeeper_state_factory.) conf auth-conf acls context))
  
 +(defn -mkStore [this conf auth-conf, acls, context]
 +  ;; This is only used by the supervisor (this does not support pacemaker)!!!
 +  (ZKStateStorage. conf, auth-conf, acls, context))
 +
  (def max-retries 10)
  
+ (defn- delete-worker-hb  [path pacemaker-client]
+   (let [response
+         (.send pacemaker-client
+                (HBMessage. HBServerMessageType/DELETE_PATH
+                            (HBMessageData/path path)))]
+     (if (= (.get_type response) HBServerMessageType/DELETE_PATH_RESPONSE)
+       :ok
+       (throw (HBExecutionException. "Invalid Response Type")))))
+ 
+ (defn- get-worker-hb [path pacemaker-client]
+   (let [response (.send pacemaker-client
+                         (HBMessage. HBServerMessageType/GET_PULSE
+                                     (HBMessageData/path path)))]
+     (if (= (.get_type response) HBServerMessageType/GET_PULSE_RESPONSE)
+       (try
+         (.get_details (.get_pulse (.get_data response)))
+         (catch Exception e
+           (throw (HBExecutionException. (.toString e)))))
+       (throw (HBExecutionException. "Invalid Response Type")))))
+ 
+ (defn- get-worker-hb-children [path pacemaker-client]
+   (let [response (.send pacemaker-client
+                         (HBMessage. HBServerMessageType/GET_ALL_NODES_FOR_PATH
+                                     (HBMessageData/path path)))]
+     (if (= (.get_type response) HBServerMessageType/GET_ALL_NODES_FOR_PATH_RESPONSE)
+       (try
+         (into [] (.get_pulseIds (.get_nodes (.get_data response))))
+         (catch Exception e
+           (throw (HBExecutionException. (.toString e)))))
+       (throw (HBExecutionException. "Invalid Response Type")))))
+ 
+ (defn shuffle-servers-list [conf]
+   (shuffle (or
+             (conf PACEMAKER-SERVERS)
+             [(conf PACEMAKER-HOST)])))
+ 
+ (defn shutdown-rotate [servers client-pool]
+   ; Shutdown the client and remove from the pool
+   (when @client-pool
+     (.shutdown (get @client-pool (first @servers)))
+     (swap! client-pool dissoc (first @servers))
+   ; Rotate server list to try another write client
+     (swap! servers (fn [lst] (concat (rest lst) [(first lst)])))))
+ 
+ (defn get-pacemaker-write-client [conf servers client-pool]
+   ;; Client should be created in case of an exception or first write call
+   ;; Shutdown happens in the retry loop
+   (try 
+     (.waitUntilReady
+      (let [client (get @client-pool (first @servers))]
+        (if (nil? client)
+          (do
+            (swap! client-pool merge {(first @servers) (PacemakerClient. conf (first @servers))})
+            (get @client-pool (first @servers)))
+          client)))
+     (catch Exception e
+       (throw e))))
+ 
+ ;; So we can mock the client for testing
+ (defn makeClientPool [conf client-pool servers]
+   (let [servers @servers
+         current-pool @client-pool]
+     (swap! client-pool merge
+            (into {}
+              (map #(if (get current-pool %)
+                      nil
+                      [% (PacemakerClient. conf %)])
+                servers)))
+     client-pool))
+ 
+ ;; Used for mocking in tests
+ (defn is-connection-ready [pacemaker-client]
+   (.isReady pacemaker-client))
+ 
+ (defn pacemaker-retry-on-exception
+   "Retries specific function on exception based on retries count"
+   [tries task-description f catchfn]
+   (let [res (try {:value (f)}
+               (catch Exception e
+                 (do
+                   (if catchfn
+                     (catchfn e))
+                   (if (= 0 tries)
+                     (throw e)
+                     {:exception e}))))]
+     (if (:exception res)
+       (do
+         (log-error
+           (:exception res)
+           (str "Failed to " task-description ". Will make [" tries "] more attempts."))
+         (recur (dec tries) task-description f catchfn))
+       (do
+         (log-debug (str "Successful " task-description "."))
+         (:value res)))))
+ 
  (defn -mkState [this conf auth-conf acls context]
    (let [zk-state (makeZKState conf auth-conf acls context)
-         pacemaker-client (makeClient conf)]
+         pacemaker-client-pool (atom {})
+         ;; We shuffle the list to spread clients among servers
+         servers (atom (shuffle-servers-list conf))]
  
      (reify
        ClusterState

http://git-wip-us.apache.org/repos/asf/storm/blob/1d4d414d/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/1d4d414d/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
----------------------------------------------------------------------


[21/21] storm git commit: add STORM-2018, 2131, 2117, 2110, 2109 to CHANGELOG

Posted by ka...@apache.org.
add STORM-2018,2131,2117,2110,2109 to CHANGELOG


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/17218ea8
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/17218ea8
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/17218ea8

Branch: refs/heads/1.x-branch
Commit: 17218ea8768f2a51f5da056ee8db8721793dff9d
Parents: 0e0cae4
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Oct 28 00:59:06 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Oct 28 01:04:37 2016 +0900

----------------------------------------------------------------------
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/17218ea8/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 49418fa..9b76dc9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,9 @@
 ## 1.1.0
+ * STORM-2109: Under supervisor V2 SUPERVISOR_MEMORY_CAPACITY_MB and SUPERVISOR_CPU_CAPACITY must be Doubles
+ * STORM-2110: in supervisor v2 filter out empty command line args
+ * STORM-2117: Supervisor V2 with local mode extracts resources directory to topology root directory instead of temporary directory
+ * STORM-2131: Add blob command to worker-launcher, make stormdist directory not writeable by topo owner
+ * STORM-2018: Supervisor V2
  * STORM-2139: Let ShellBolts and ShellSpouts run with scripts from blobs
  * STORM-2072: Add map, flatMap with different outputs (T->V) in Trident
  * STORM-2134: improving the current scheduling strategy for RAS


[13/21] storm git commit: Merge branch 'STORM-2109' of https://github.com/revans2/incubator-storm into STORM-2109

Posted by ka...@apache.org.
Merge branch 'STORM-2109' of https://github.com/revans2/incubator-storm into STORM-2109


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2297757e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2297757e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2297757e

Branch: refs/heads/1.x-branch
Commit: 2297757e100e314bcf7834f403e310b8909d65a8
Parents: 88619b2
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Sun Sep 25 21:26:18 2016 +0900
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Sep 26 08:22:42 2016 -0500

----------------------------------------------------------------------
 .../storm/daemon/supervisor/timer/SupervisorHeartbeat.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/2297757e/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
index 34c5682..849c584 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
@@ -72,9 +72,9 @@ public class SupervisorHeartbeat implements Runnable {
 
     private Map<String, Double> mkSupervisorCapacities(Map conf) {
         Map<String, Double> ret = new HashMap<String, Double>();
-        Double mem = (double) (conf.get(Config.SUPERVISOR_MEMORY_CAPACITY_MB));
+        Double mem = Utils.getDouble(conf.get(Config.SUPERVISOR_MEMORY_CAPACITY_MB), 4096.0);
         ret.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, mem);
-        Double cpu = (double) (conf.get(Config.SUPERVISOR_CPU_CAPACITY));
+        Double cpu = Utils.getDouble(conf.get(Config.SUPERVISOR_CPU_CAPACITY), 400.0);
         ret.put(Config.SUPERVISOR_CPU_CAPACITY, cpu);
         return ret;
     }


[12/21] storm git commit: Merge branch 'STORM-2110' of https://github.com/revans2/incubator-storm into STORM-2110

Posted by ka...@apache.org.
Merge branch 'STORM-2110' of https://github.com/revans2/incubator-storm into STORM-2110

Conflicts:
	storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/88619b21
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/88619b21
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/88619b21

Branch: refs/heads/1.x-branch
Commit: 88619b2126c9877f7d35b47ba5648cb006637721
Parents: fb42b96
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Sep 23 10:43:33 2016 +0900
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Sep 23 13:00:20 2016 -0500

----------------------------------------------------------------------
 .../jvm/org/apache/storm/daemon/supervisor/BasicContainer.java | 6 +++++-
 .../org/apache/storm/daemon/supervisor/BasicContainerTest.java | 2 ++
 2 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/88619b21/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
index c5b40f2..b1fa226 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -412,7 +412,11 @@ public class BasicContainer extends Container {
             String string = substituteChildOptsInternal((String) value, memOnheap);
             if (StringUtils.isNotBlank(string)) {
                 String[] strings = string.split("\\s+");
-                rets.addAll(Arrays.asList(strings));
+                for (String s: strings) {
+                    if (StringUtils.isNotBlank(s)) {
+                        rets.add(s);
+                    }
+                }
             }
         } else if (value instanceof List) {
             @SuppressWarnings("unchecked")

http://git-wip-us.apache.org/repos/asf/storm/blob/88619b21/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
index 38081f4..3e41125 100644
--- a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
@@ -381,6 +381,7 @@ public class BasicContainerTest {
         superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
         superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
         superConf.put(Config.STORM_LOG4J2_CONF_DIR, log4jdir);
+        superConf.put(Config.WORKER_CHILDOPTS, " -Dtesting=true");
         
         final LocalAssignment la = new LocalAssignment();
         la.set_topology_id(topoId);
@@ -432,6 +433,7 @@ public class BasicContainerTest {
                     "-Dlog4j.configurationFile=" + workerConf,
                     "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
                     "-Dstorm.local.dir=" + stormLocal,
+                    "-Dtesting=true",
                     "-Djava.library.path=JLP",
                     "-Dstorm.conf.file=",
                     "-Dstorm.options=",


[04/21] storm git commit: STORM-2018: Supervisor V2

Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java
new file mode 100644
index 0000000..d33dc9c
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/testing/staticmocking/MockedSupervisorUtils.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version
+ * 2.0 (the "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.testing.staticmocking;
+
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+
+public class MockedSupervisorUtils implements AutoCloseable {
+
+    public MockedSupervisorUtils(SupervisorUtils inst) {
+        SupervisorUtils.setInstance(inst);
+    }
+
+    @Override
+    public void close() throws Exception {
+        SupervisorUtils.resetInstance();
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/trident/util/TridentUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/trident/util/TridentUtils.java b/storm-core/src/jvm/org/apache/storm/trident/util/TridentUtils.java
index 8272b3c..f5d317e 100644
--- a/storm-core/src/jvm/org/apache/storm/trident/util/TridentUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/trident/util/TridentUtils.java
@@ -111,7 +111,7 @@ public class TridentUtils {
         return Utils.thriftSerialize(t);
     }
 
-    public static <T> T thriftDeserialize(Class c, byte[] b) {
+    public static <T> T thriftDeserialize(Class<T> c, byte[] b) {
         return Utils.thriftDeserialize(c,b);
     }
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java b/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java
new file mode 100644
index 0000000..e2be8a7
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/utils/ConfigUtils.java
@@ -0,0 +1,536 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.utils;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.storm.Config;
+import org.apache.storm.daemon.supervisor.AdvancedFSOps;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.validation.ConfigValidation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class ConfigUtils {
+    private final static Logger LOG = LoggerFactory.getLogger(ConfigUtils.class);
+    public final static String RESOURCES_SUBDIR = "resources";
+    public final static String NIMBUS_DO_NOT_REASSIGN = "NIMBUS-DO-NOT-REASSIGN";
+    public static final String FILE_SEPARATOR = File.separator;
+
+    // A singleton instance allows us to mock delegated static methods in our
+    // tests by subclassing.
+    private static ConfigUtils _instance = new ConfigUtils();
+
+    /**
+     * Provide an instance of this class for delegates to use.  To mock out
+     * delegated methods, provide an instance of a subclass that overrides the
+     * implementation of the delegated method.
+     * @param u a Utils instance
+     * @return the previously set instance
+     */
+    public static ConfigUtils setInstance(ConfigUtils u) {
+        ConfigUtils oldInstance = _instance;
+        _instance = u;
+        return oldInstance;
+    }
+
+    public static String getLogDir() {
+        String dir;
+        Map conf;
+        if (System.getProperty("storm.log.dir") != null) {
+            dir = System.getProperty("storm.log.dir");
+        } else if ((conf = readStormConfig()).get("storm.log.dir") != null) {
+            dir = String.valueOf(conf.get("storm.log.dir"));
+        } else if (System.getProperty("storm.home") != null) {
+            dir = System.getProperty("storm.home") + FILE_SEPARATOR + "logs";
+        } else {
+            dir = "logs";
+        }
+        try {
+            return new File(dir).getCanonicalPath();
+        } catch (IOException ex) {
+            throw new IllegalArgumentException("Illegal storm.log.dir in conf: " + dir);
+        }
+    }
+
+    public static String clojureConfigName(String name) {
+        return name.toUpperCase().replace("_", "-");
+    }
+
+    // ALL-CONFIGS is only used by executor.clj once, do we want to do it here? TODO
+    public static List<Object> All_CONFIGS() {
+        List<Object> ret = new ArrayList<Object>();
+        Config config = new Config();
+        Class<?> ConfigClass = config.getClass();
+        Field[] fields = ConfigClass.getFields();
+        for (int i = 0; i < fields.length; i++) {
+            try {
+                Object obj = fields[i].get(null);
+                ret.add(obj);
+            } catch (IllegalArgumentException e) {
+                LOG.error(e.getMessage(), e);
+            } catch (IllegalAccessException e) {
+                LOG.error(e.getMessage(), e);
+            }
+        }
+        return ret;
+    }
+
+    public static String clusterMode(Map conf) {
+        String mode = (String) conf.get(Config.STORM_CLUSTER_MODE);
+        return mode;
+    }
+
+    public static boolean isLocalMode(Map<String, Object> conf) {
+        String mode = (String) conf.get(Config.STORM_CLUSTER_MODE);
+        if (mode != null) {
+            if ("local".equals(mode)) {
+                return true;
+            }
+            if ("distributed".equals(mode)) {
+                return false;
+            }
+            throw new IllegalArgumentException("Illegal cluster mode in conf: " + mode);
+        }
+        return true;
+    }
+
+    public static int samplingRate(Map conf) {
+        double rate = Utils.getDouble(conf.get(Config.TOPOLOGY_STATS_SAMPLE_RATE));
+        if (rate != 0) {
+            return (int) (1 / rate);
+        }
+        throw new IllegalArgumentException("Illegal topology.stats.sample.rate in conf: " + rate);
+    }
+
+    public static Callable<Boolean> evenSampler(final int samplingFreq) {
+        final Random random = new Random();
+
+        return new Callable<Boolean>() {
+            private int curr = -1;
+            private int target = random.nextInt(samplingFreq);
+
+            @Override
+            public Boolean call() throws Exception {
+                curr++;
+                if (curr >= samplingFreq) {
+                    curr = 0;
+                    target = random.nextInt(samplingFreq);
+                }
+                return (curr == target);
+            }
+        };
+    }
+
+    public static Callable<Boolean> mkStatsSampler(Map conf) {
+        return evenSampler(samplingRate(conf));
+    }
+
+    // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+    public static Map<String, Object> readStormConfig() {
+        return _instance.readStormConfigImpl();
+    }
+
+    public Map<String, Object> readStormConfigImpl() {
+        Map<String, Object> conf = Utils.readStormConfig();
+        ConfigValidation.validateFields(conf);
+        return conf;
+    }
+
+    public static Map readYamlConfig(String name, boolean mustExist) {
+        Map conf = Utils.findAndReadConfigFile(name, mustExist);
+        ConfigValidation.validateFields(conf);
+        return conf;
+    }
+
+    public static Map readYamlConfig(String name) {
+        return readYamlConfig(name, true);
+    }
+
+    public static String absoluteStormLocalDir(Map conf) {
+        String stormHome = System.getProperty("storm.home");
+        String localDir = (String) conf.get(Config.STORM_LOCAL_DIR);
+        if (localDir == null) {
+            return (stormHome + FILE_SEPARATOR + "storm-local");
+        } else {
+            if (new File(localDir).isAbsolute()) {
+                return localDir;
+            } else {
+                return (stormHome + FILE_SEPARATOR + localDir);
+            }
+        }
+    }
+
+    public static String absoluteHealthCheckDir(Map conf) {
+        String stormHome = System.getProperty("storm.home");
+        String healthCheckDir = (String) conf.get(Config.STORM_HEALTH_CHECK_DIR);
+        if (healthCheckDir == null) {
+            return (stormHome + FILE_SEPARATOR + "healthchecks");
+        } else {
+            if (new File(healthCheckDir).isAbsolute()) {
+                return healthCheckDir;
+            } else {
+                return (stormHome + FILE_SEPARATOR + healthCheckDir);
+            }
+        }
+    }
+
+    public static String masterLocalDir(Map conf) throws IOException {
+        String ret = String.valueOf(conf.get(Config.STORM_LOCAL_DIR)) + FILE_SEPARATOR + "nimbus";
+        FileUtils.forceMkdir(new File(ret));
+        return ret;
+    }
+
+    public static String masterStormJarKey(String topologyId) {
+        return (topologyId + "-stormjar.jar");
+    }
+
+    public static String masterStormCodeKey(String topologyId) {
+        return (topologyId + "-stormcode.ser");
+    }
+
+    public static String masterStormConfKey(String topologyId) {
+        return (topologyId + "-stormconf.ser");
+    }
+
+    public static String masterStormDistRoot(Map conf) throws IOException {
+        String ret = stormDistPath(masterLocalDir(conf));
+        FileUtils.forceMkdir(new File(ret));
+        return ret;
+    }
+
+    public static String masterStormDistRoot(Map conf, String stormId) throws IOException {
+        return (masterStormDistRoot(conf) + FILE_SEPARATOR + stormId);
+    }
+
+    public static String stormDistPath(String stormRoot) {
+        String ret = "";
+        // we do this since to concat a null String will actually concat a "null", which is not the expected: ""
+        if (stormRoot != null) {
+            ret = stormRoot;
+        }
+        return ret + FILE_SEPARATOR + "stormdist";
+    }
+
+    public static Map<String, Object> readSupervisorStormConfGivenPath(Map<String, Object> conf, String stormConfPath) throws IOException {
+        Map<String, Object> ret = new HashMap<>(conf);
+        ret.putAll(Utils.fromCompressedJsonConf(FileUtils.readFileToByteArray(new File(stormConfPath))));
+        return ret;
+    }
+
+    public static StormTopology readSupervisorStormCodeGivenPath(String stormCodePath, AdvancedFSOps ops) throws IOException {
+        return Utils.deserialize(ops.slurp(new File(stormCodePath)), StormTopology.class);
+    }
+
+    public static String masterStormJarPath(String stormRoot) {
+        return (stormRoot + FILE_SEPARATOR + "stormjar.jar");
+    }
+
+    public static String masterInbox(Map conf) throws IOException {
+        String ret = masterLocalDir(conf) + FILE_SEPARATOR + "inbox";
+        FileUtils.forceMkdir(new File(ret));
+        return ret;
+    }
+
+    public static String masterInimbusDir(Map conf) throws IOException {
+        return (masterLocalDir(conf) + FILE_SEPARATOR + "inimbus");
+    }
+
+    // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+    public static String supervisorLocalDir(Map conf) throws IOException {
+        return _instance.supervisorLocalDirImpl(conf);
+    }
+
+    public String supervisorLocalDirImpl(Map conf) throws IOException {
+        String ret = absoluteStormLocalDir(conf) + FILE_SEPARATOR + "supervisor";
+        FileUtils.forceMkdir(new File(ret));
+        return ret;
+    }
+
+    public static String supervisorIsupervisorDir(Map conf) throws IOException {
+        return (supervisorLocalDir(conf) + FILE_SEPARATOR + "isupervisor");
+    }
+
+    // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+    public static String supervisorStormDistRoot(Map conf) throws IOException {
+        return _instance.supervisorStormDistRootImpl(conf);
+    }
+
+    public String supervisorStormDistRootImpl(Map conf) throws IOException {
+        return stormDistPath(supervisorLocalDir(conf));
+    }
+
+    // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+    public static String supervisorStormDistRoot(Map conf, String stormId) throws IOException {
+        return _instance.supervisorStormDistRootImpl(conf, stormId);
+    }
+
+    public String supervisorStormDistRootImpl(Map conf, String stormId) throws IOException {
+        return supervisorStormDistRoot(conf) + FILE_SEPARATOR + URLEncoder.encode(stormId, "UTF-8");
+    }
+
+    public static String concatIfNotNull(String dir) {
+        String ret = "";
+        // we do this since to concat a null String will actually concat a "null", which is not the expected: ""
+        if (dir != null) {
+            ret = dir;
+        }
+        return ret;
+    }
+
+    public static String supervisorStormJarPath(String stormRoot) {
+        return (concatIfNotNull(stormRoot) + FILE_SEPARATOR + "stormjar.jar");
+    }
+
+    public static String supervisorStormCodePath(String stormRoot) {
+        return (concatIfNotNull(stormRoot) + FILE_SEPARATOR + "stormcode.ser");
+    }
+
+    public static String supervisorStormConfPath(String stormRoot) {
+        return (concatIfNotNull(stormRoot) + FILE_SEPARATOR + "stormconf.ser");
+    }
+
+    public static String supervisorTmpDir(Map conf) throws IOException {
+        String ret = supervisorLocalDir(conf) + FILE_SEPARATOR + "tmp";
+        FileUtils.forceMkdir(new File(ret));
+        return ret;
+    }
+
+    public static String supervisorStormResourcesPath(String stormRoot) {
+        return (concatIfNotNull(stormRoot) + FILE_SEPARATOR + RESOURCES_SUBDIR);
+    }
+
+    // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+    public static LocalState supervisorState(Map conf) throws IOException {
+        return _instance.supervisorStateImpl(conf);
+    }
+
+    public LocalState supervisorStateImpl(Map conf) throws IOException {
+        return new LocalState((supervisorLocalDir(conf) + FILE_SEPARATOR + "localstate"));
+    }
+
+    // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+    public static LocalState nimbusTopoHistoryState(Map conf) throws IOException {
+        return _instance.nimbusTopoHistoryStateImpl(conf);
+    }
+
+    public LocalState nimbusTopoHistoryStateImpl(Map conf) throws IOException {
+        return new LocalState((masterLocalDir(conf) + FILE_SEPARATOR + "history"));
+    }
+
+    // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+    public static Map<String, Object> readSupervisorStormConf(Map<String, Object> conf, String stormId) throws IOException {
+        return _instance.readSupervisorStormConfImpl(conf, stormId);
+    }
+
+    public Map<String, Object> readSupervisorStormConfImpl(Map<String, Object> conf, String stormId) throws IOException {
+        String stormRoot = supervisorStormDistRoot(conf, stormId);
+        String confPath = supervisorStormConfPath(stormRoot);
+        return readSupervisorStormConfGivenPath(conf, confPath);
+    }
+
+    public static StormTopology readSupervisorTopology(Map conf, String stormId, AdvancedFSOps ops) throws IOException {
+        return _instance.readSupervisorTopologyImpl(conf, stormId, ops);
+    }
+  
+    public StormTopology readSupervisorTopologyImpl(Map conf, String stormId, AdvancedFSOps ops) throws IOException {
+        String stormRoot = supervisorStormDistRoot(conf, stormId);
+        String topologyPath = supervisorStormCodePath(stormRoot);
+        return readSupervisorStormCodeGivenPath(topologyPath, ops);
+    }
+
+    public static String workerUserRoot(Map conf) {
+        return (absoluteStormLocalDir(conf) + FILE_SEPARATOR + "workers-users");
+    }
+
+    public static String workerUserFile(Map conf, String workerId) {
+        return (workerUserRoot(conf) + FILE_SEPARATOR + workerId);
+    }
+
+    public static String getIdFromBlobKey(String key) {
+        if (key == null) return null;
+        final String STORM_JAR_SUFFIX = "-stormjar.jar";
+        final String STORM_CODE_SUFFIX = "-stormcode.ser";
+        final String STORM_CONF_SUFFIX = "-stormconf.ser";
+
+        String ret = null;
+        if (key.endsWith(STORM_JAR_SUFFIX)) {
+            ret = key.substring(0, key.length() - STORM_JAR_SUFFIX.length());
+        } else if (key.endsWith(STORM_CODE_SUFFIX)) {
+            ret = key.substring(0, key.length() - STORM_CODE_SUFFIX.length());
+        } else if (key.endsWith(STORM_CONF_SUFFIX)) {
+            ret = key.substring(0, key.length() - STORM_CONF_SUFFIX.length());
+        }
+        return ret;
+    }
+
+    // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+    public static String workerArtifactsRoot(Map conf) {
+        return _instance.workerArtifactsRootImpl(conf);
+    }
+
+    public String workerArtifactsRootImpl(Map conf) {
+        String artifactsDir = (String)conf.get(Config.STORM_WORKERS_ARTIFACTS_DIR);
+        if (artifactsDir == null) {
+            return (getLogDir() + FILE_SEPARATOR + "workers-artifacts");
+        } else {
+            if (new File(artifactsDir).isAbsolute()) {
+                return artifactsDir;
+            } else {
+                return (getLogDir() + FILE_SEPARATOR + artifactsDir);
+            }
+        }
+    }
+
+    public static String workerArtifactsRoot(Map conf, String id) {
+        return (workerArtifactsRoot(conf) + FILE_SEPARATOR + id);
+    }
+
+    public static String workerArtifactsRoot(Map conf, String id, Integer port) {
+        return (workerArtifactsRoot(conf, id) + FILE_SEPARATOR + port);
+    }
+
+    public static String workerArtifactsPidPath(Map conf, String id, Integer port) {
+        return (workerArtifactsRoot(conf, id, port) + FILE_SEPARATOR +  "worker.pid");
+    }
+
+    public static File getLogMetaDataFile(String fname) {
+        String[] subStrings = fname.split(FILE_SEPARATOR); // TODO: does this work well on windows?
+        String id = subStrings[0];
+        Integer port = Integer.parseInt(subStrings[1]);
+        return getLogMetaDataFile(Utils.readStormConfig(), id, port);
+    }
+
+    public static File getLogMetaDataFile(Map conf, String id, Integer port) {
+        String fname = workerArtifactsRoot(conf, id, port) + FILE_SEPARATOR + "worker.yaml";
+        return new File(fname);
+    }
+
+    public static File getWorkerDirFromRoot(String logRoot, String id, Integer port) {
+        return new File((logRoot + FILE_SEPARATOR + id + FILE_SEPARATOR + port));
+    }
+
+    // we use this "weird" wrapper pattern temporarily for mocking in clojure test
+    public static String workerRoot(Map conf) {
+        return _instance.workerRootImpl(conf);
+    }
+
+    public String workerRootImpl(Map conf) {
+        return (absoluteStormLocalDir(conf) + FILE_SEPARATOR + "workers");
+    }
+
+    public static String workerRoot(Map conf, String id) {
+        return (workerRoot(conf) + FILE_SEPARATOR + id);
+    }
+
+    public static String workerPidsRoot(Map conf, String id) {
+        return (workerRoot(conf, id) + FILE_SEPARATOR + "pids");
+    }
+
+    public static String workerTmpRoot(Map conf, String id) {
+        return (workerRoot(conf, id) + FILE_SEPARATOR + "tmp");
+    }
+
+
+    public static String workerPidPath(Map conf, String id, String pid) {
+        return (workerPidsRoot(conf, id) + FILE_SEPARATOR + pid);
+    }
+    
+    public static String workerPidPath(Map<String, Object> conf, String id, long pid) {
+        return workerPidPath(conf, id, String.valueOf(pid));
+    }
+
+    public static String workerHeartbeatsRoot(Map conf, String id) {
+        return (workerRoot(conf, id) + FILE_SEPARATOR + "heartbeats");
+    }
+
+    public static LocalState workerState(Map conf, String id) throws IOException {
+        return new LocalState(workerHeartbeatsRoot(conf, id));
+    }
+
+    public static Map overrideLoginConfigWithSystemProperty(Map conf) { // note that we delete the return value
+        String loginConfFile = System.getProperty("java.security.auth.login.config");
+        if (loginConfFile != null) {
+             conf.put("java.security.auth.login.config", loginConfFile);
+        }
+        return conf;
+    }
+
+    /* TODO: make sure test these two functions in manual tests */
+    public static List<String> getTopoLogsUsers(Map topologyConf) {
+        List<String> logsUsers = (List<String>)topologyConf.get(Config.LOGS_USERS);
+        List<String> topologyUsers = (List<String>)topologyConf.get(Config.TOPOLOGY_USERS);
+        Set<String> mergedUsers = new HashSet<String>();
+        if (logsUsers != null) {
+            for (String user : logsUsers) {
+                if (user != null) {
+                    mergedUsers.add(user);
+                }
+            }
+        }
+        if (topologyUsers != null) {
+            for (String user : topologyUsers) {
+                if (user != null) {
+                    mergedUsers.add(user);
+                }
+            }
+        }
+        List<String> ret = new ArrayList<String>(mergedUsers);
+        Collections.sort(ret);
+        return ret;
+    }
+
+    public static List<String> getTopoLogsGroups(Map topologyConf) {
+        List<String> logsGroups = (List<String>)topologyConf.get(Config.LOGS_GROUPS);
+        List<String> topologyGroups = (List<String>)topologyConf.get(Config.TOPOLOGY_GROUPS);
+        Set<String> mergedGroups = new HashSet<String>();
+        if (logsGroups != null) {
+            for (String group : logsGroups) {
+                if (group != null) {
+                    mergedGroups.add(group);
+                }
+            }
+        }
+        if (topologyGroups != null) {
+            for (String group : topologyGroups) {
+                if (group != null) {
+                    mergedGroups.add(group);
+                }
+            }
+        }
+        List<String> ret = new ArrayList<String>(mergedGroups);
+        Collections.sort(ret);
+        return ret;
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/InprocMessaging.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/InprocMessaging.java b/storm-core/src/jvm/org/apache/storm/utils/InprocMessaging.java
index 51250f4..8583e0d 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/InprocMessaging.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/InprocMessaging.java
@@ -19,41 +19,82 @@ package org.apache.storm.utils;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class InprocMessaging {
-    private static Map<Integer, LinkedBlockingQueue<Object>> _queues = new HashMap<>();
-    private static final Object _lock = new Object();
+    private static Map<Integer, LinkedBlockingQueue<Object>> _queues = new HashMap<Integer, LinkedBlockingQueue<Object>>();
+    private static ConcurrentMap<Integer, AtomicBoolean> _hasReader = new ConcurrentHashMap<>();
     private static int port = 1;
+    private static final Logger LOG = LoggerFactory.getLogger(InprocMessaging.class);
     
-    public static int acquireNewPort() {
-        int ret;
-        synchronized(_lock) {
-            ret = port;
-            port++;
-        }
+    public synchronized static int acquireNewPort() {
+        int ret = port;
+        port++;
         return ret;
     }
     
     public static void sendMessage(int port, Object msg) {
+        waitForReader(port);
+        getQueue(port).add(msg);
+    }
+    
+    public static void sendMessageNoWait(int port, Object msg) {
         getQueue(port).add(msg);
     }
     
     public static Object takeMessage(int port) throws InterruptedException {
+        readerArrived(port);
         return getQueue(port).take();
     }
 
     public static Object pollMessage(int port) {
+        readerArrived(port);
         return  getQueue(port).poll();
-    }    
+    }
+    
+    private static AtomicBoolean getHasReader(int port) {
+        AtomicBoolean ab = _hasReader.get(port);
+        if (ab == null) {
+            _hasReader.putIfAbsent(port, new AtomicBoolean(false));
+            ab = _hasReader.get(port);
+        }
+        return ab;
+    }
     
-    private static LinkedBlockingQueue<Object> getQueue(int port) {
-        synchronized(_lock) {
-            if(!_queues.containsKey(port)) {
-              _queues.put(port, new LinkedBlockingQueue<>());
+    public static void waitForReader(int port) {
+        AtomicBoolean ab = getHasReader(port);
+        long start = Time.currentTimeMillis();
+        while (!ab.get()) {
+            if (Time.isSimulating()) {
+                Time.advanceTime(100);
+            }
+            try {
+                Thread.sleep(10);
+            } catch (InterruptedException e) {
+                //Ignored
             }
-            return _queues.get(port);
+            if (Time.currentTimeMillis() - start > 20000) {
+                LOG.error("DONE WAITING FOR READER AFTER {} ms", Time.currentTimeMillis() - start);
+                break;
+            }
+        }
+    }
+    
+    private static void readerArrived(int port) {
+        getHasReader(port).set(true);
+    }
+    
+    private synchronized static LinkedBlockingQueue<Object> getQueue(int port) {
+        if(!_queues.containsKey(port)) {
+            _queues.put(port, new LinkedBlockingQueue<Object>());
         }
+        return _queues.get(port);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/LocalState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/LocalState.java b/storm-core/src/jvm/org/apache/storm/utils/LocalState.java
index aef1c1c..2f0bb60 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/LocalState.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/LocalState.java
@@ -18,24 +18,28 @@
 package org.apache.storm.utils;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.storm.generated.LSApprovedWorkers;
+import org.apache.storm.generated.LSSupervisorAssignments;
+import org.apache.storm.generated.LSSupervisorId;
+import org.apache.storm.generated.LSTopoHistory;
+import org.apache.storm.generated.LSTopoHistoryList;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.LocalStateData;
+import org.apache.storm.generated.ThriftSerializedObject;
+import org.apache.thrift.TBase;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TSerializer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.Map;
+import java.util.ArrayList;
 import java.util.HashMap;
-import java.io.IOException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.thrift.TBase;
-import org.apache.thrift.TDeserializer;
-import org.apache.thrift.TException;
-import org.apache.thrift.TSerializer;
-
-import org.apache.storm.generated.LocalStateData;
-import org.apache.storm.generated.ThriftSerializedObject;
+import java.util.List;
+import java.util.Map;
 
 /**
  * A simple, durable, atomic K/V database. *Very inefficient*, should only be used for occasional reads/writes.
@@ -43,6 +47,11 @@ import org.apache.storm.generated.ThriftSerializedObject;
  */
 public class LocalState {
     public static final Logger LOG = LoggerFactory.getLogger(LocalState.class);
+    public static final String LS_WORKER_HEARTBEAT = "worker-heartbeat";
+    public static final String LS_ID = "supervisor-id";
+    public static final String LS_LOCAL_ASSIGNMENTS = "local-assignments";
+    public static final String LS_APPROVED_WORKERS = "approved-workers";
+    public static final String LS_TOPO_HISTORY = "topo-hist";
     private VersionedStore _vs;
     
     public LocalState(String backingDir) throws IOException {
@@ -157,6 +166,85 @@ public class LocalState {
         _vs.cleanup(keepVersions);
     }
 
+    public List<LSTopoHistory> getTopoHistoryList() {
+        LSTopoHistoryList lsTopoHistoryListWrapper = (LSTopoHistoryList) get(LS_TOPO_HISTORY);
+        if (null != lsTopoHistoryListWrapper) {
+            return lsTopoHistoryListWrapper.get_topo_history();
+        }
+        return null;
+    }
+
+    /**
+     * Remove topologies from local state which are older than cutOffAge.
+     * @param cutOffAge
+     */
+    public void filterOldTopologies(long cutOffAge) {
+        LSTopoHistoryList lsTopoHistoryListWrapper = (LSTopoHistoryList) get(LS_TOPO_HISTORY);
+        List<LSTopoHistory> filteredTopoHistoryList = new ArrayList<>();
+        if (null != lsTopoHistoryListWrapper) {
+            for (LSTopoHistory topoHistory : lsTopoHistoryListWrapper.get_topo_history()) {
+                if (topoHistory.get_time_stamp() > cutOffAge) {
+                    filteredTopoHistoryList.add(topoHistory);
+                }
+            }
+        }
+        put(LS_TOPO_HISTORY, new LSTopoHistoryList(filteredTopoHistoryList));
+    }
+
+    public void addTopologyHistory(LSTopoHistory lsTopoHistory) {
+        LSTopoHistoryList lsTopoHistoryListWrapper = (LSTopoHistoryList) get(LS_TOPO_HISTORY);
+        List<LSTopoHistory> currentTopoHistoryList = new ArrayList<>();
+        if (null != lsTopoHistoryListWrapper) {
+            currentTopoHistoryList.addAll(lsTopoHistoryListWrapper.get_topo_history());
+        }
+        currentTopoHistoryList.add(lsTopoHistory);
+        put(LS_TOPO_HISTORY, new LSTopoHistoryList(currentTopoHistoryList));
+    }
+
+    public String getSupervisorId() {
+        LSSupervisorId lsSupervisorId = (LSSupervisorId) get(LS_ID);
+        if (null != lsSupervisorId) {
+            return lsSupervisorId.get_supervisor_id();
+        }
+        return null;
+    }
+
+    public void setSupervisorId(String supervisorId) {
+        put(LS_ID, new LSSupervisorId(supervisorId));
+    }
+
+    public Map<String, Integer> getApprovedWorkers() {
+        LSApprovedWorkers lsApprovedWorkers = (LSApprovedWorkers) get(LS_APPROVED_WORKERS);
+        if (null != lsApprovedWorkers) {
+            return lsApprovedWorkers.get_approved_workers();
+        }
+        return null;
+    }
+
+    public void setApprovedWorkers(Map<String, Integer> approvedWorkers) {
+        put(LS_APPROVED_WORKERS, new LSApprovedWorkers(approvedWorkers));
+    }
+
+    public LSWorkerHeartbeat getWorkerHeartBeat() {
+        return (LSWorkerHeartbeat) get(LS_WORKER_HEARTBEAT);
+    }
+
+    public void setWorkerHeartBeat(LSWorkerHeartbeat workerHeartBeat) {
+        put(LS_WORKER_HEARTBEAT, workerHeartBeat, false);
+    }
+
+    public Map<Integer, LocalAssignment> getLocalAssignmentsMap() {
+        LSSupervisorAssignments assignments = (LSSupervisorAssignments) get(LS_LOCAL_ASSIGNMENTS);
+        if (null != assignments) {
+            return assignments.get_assignments();
+        }
+        return null;
+    }
+
+    public void setLocalAssignmentsMap(Map<Integer, LocalAssignment> localAssignmentMap) {
+        put(LS_LOCAL_ASSIGNMENTS, new LSSupervisorAssignments(localAssignmentMap));
+    }
+
     private void persistInternal(Map<String, ThriftSerializedObject> serialized, TSerializer ser, boolean cleanup) {
         try {
             if (ser == null) {

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
index 456b459..0e5cef9 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/NimbusClient.java
@@ -17,11 +17,11 @@
  */
 package org.apache.storm.utils;
 
-
 import org.apache.storm.Config;
 import org.apache.storm.generated.ClusterSummary;
 import org.apache.storm.generated.Nimbus;
 import org.apache.storm.generated.NimbusSummary;
+import org.apache.storm.security.auth.ReqContext;
 import org.apache.storm.security.auth.ThriftClient;
 import org.apache.storm.security.auth.ThriftConnectionType;
 import com.google.common.collect.Lists;
@@ -29,6 +29,7 @@ import org.apache.thrift.transport.TTransportException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.security.Principal;
 import java.util.List;
 import java.util.Map;
 
@@ -36,6 +37,22 @@ public class NimbusClient extends ThriftClient {
     private Nimbus.Client _client;
     private static final Logger LOG = LoggerFactory.getLogger(NimbusClient.class);
 
+    public interface WithNimbus {
+        public void run(Nimbus.Client client) throws Exception;
+    }
+
+    public static void withConfiguredClient(WithNimbus cb) throws Exception {
+        withConfiguredClient(cb, ConfigUtils.readStormConfig());
+    }
+
+    public static void withConfiguredClient(WithNimbus cb, Map conf) throws Exception {
+        ReqContext context = ReqContext.context();
+        Principal principal = context.principal();
+        String user = principal == null ? null : principal.getName();
+        try (NimbusClient client = getConfiguredClientAs(conf, user);) {
+            cb.run(client.getClient());
+        }
+    }
 
     public static NimbusClient getConfiguredClient(Map conf) {
         return getConfiguredClientAs(conf, null);

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/utils/Time.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/utils/Time.java b/storm-core/src/jvm/org/apache/storm/utils/Time.java
index 1792252..e501b6c 100644
--- a/storm-core/src/jvm/org/apache/storm/utils/Time.java
+++ b/storm-core/src/jvm/org/apache/storm/utils/Time.java
@@ -29,6 +29,7 @@ public class Time {
     public static final Logger LOG = LoggerFactory.getLogger(Time.class);
     
     private static AtomicBoolean simulating = new AtomicBoolean(false);
+    private static AtomicLong autoAdvanceOnSleep = new AtomicLong(0);
     //TODO: should probably use weak references here or something
     private static volatile Map<Thread, AtomicLong> threadSleepTimes;
     private static final Object sleepTimesLock = new Object();
@@ -43,10 +44,18 @@ public class Time {
         }
     }
     
+    public static void startSimulatingAutoAdvanceOnSleep(long ms) {
+        synchronized(sleepTimesLock) {
+            startSimulating();
+            autoAdvanceOnSleep.set(ms);
+        }
+    }
+    
     public static void stopSimulating() {
         synchronized(sleepTimesLock) {
-            simulating.set(false);             
-            threadSleepTimes = null;  
+            simulating.set(false);    
+            autoAdvanceOnSleep.set(0);
+            threadSleepTimes = null;
         }
     }
     
@@ -71,6 +80,10 @@ public class Time {
                             throw new InterruptedException();
                         }
                     }
+                    long autoAdvance = autoAdvanceOnSleep.get();
+                    if (autoAdvance > 0) {
+                        advanceTime(autoAdvance);
+                    }
                     Thread.sleep(10);
                 }
             } finally {
@@ -86,10 +99,16 @@ public class Time {
                 Thread.sleep(sleepTime);
         }
     }
-    
+
     public static void sleep(long ms) throws InterruptedException {
         sleepUntil(currentTimeMillis()+ms);
     }
+
+    public static void sleepSecs (long secs) throws InterruptedException {
+        if (secs > 0) {
+            sleep(secs * 1000);
+        }
+    }
     
     public static long currentTimeMillis() {
         if(simulating.get()) {
@@ -98,14 +117,32 @@ public class Time {
             return System.currentTimeMillis();
         }
     }
-    
+
+    public static long secsToMillis (int secs) {
+        return 1000*(long) secs;
+    }
+
+    public static long secsToMillisLong(double secs) {
+        return (long) (1000 * secs);
+    }
+
     public static int currentTimeSecs() {
         return (int) (currentTimeMillis() / 1000);
     }
+
+    public static int deltaSecs(int timeInSeconds) {
+        return Time.currentTimeSecs() - timeInSeconds;
+    }
+
+    public static long deltaMs(long timeInMilliseconds) {
+        return Time.currentTimeMillis() - timeInMilliseconds;
+    }
     
     public static void advanceTime(long ms) {
-        if(!simulating.get()) throw new IllegalStateException("Cannot simulate time unless in simulation mode");
-        simulatedCurrTimeMs.set(simulatedCurrTimeMs.get() + ms);
+        if (!simulating.get()) throw new IllegalStateException("Cannot simulate time unless in simulation mode");
+        if (ms < 0) throw new IllegalArgumentException("advanceTime only accepts positive time as an argument");
+        long newTime = simulatedCurrTimeMs.addAndGet(ms);
+        LOG.warn("Advanced simulated time to {}", newTime);
     }
     
     public static boolean isThreadWaiting(Thread t) {
@@ -115,5 +152,5 @@ public class Time {
             time = threadSleepTimes.get(t);
         }
         return !t.isAlive() || time!=null && currentTimeMillis() < time.longValue();
-    }    
+    }
 }


[16/21] storm git commit: Another java 7 fix

Posted by ka...@apache.org.
Another java 7 fix


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f12b7dae
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f12b7dae
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f12b7dae

Branch: refs/heads/1.x-branch
Commit: f12b7daeb759002f8a578c32c3fd70c1315644c7
Parents: 905c443
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Sep 26 09:54:03 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Sep 26 09:54:03 2016 -0500

----------------------------------------------------------------------
 .../src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/f12b7dae/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
index 977cb2b..daa1d00 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -529,7 +529,7 @@ public class BasicContainer extends Container {
             //Only go off of the topology id for now.
             DependencyLocations dl = _cache.get(topologyId);
             if (dl == null) {
-                _cache.putIfAbsent(topologyId, new DependencyLocations(conf, topologyId, ops, stormRoot));
+                _cache.put(topologyId, new DependencyLocations(conf, topologyId, ops, stormRoot));
                 dl = _cache.get(topologyId);
             }
             return dl;


[05/21] storm git commit: STORM-2018: Supervisor V2

Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
new file mode 100644
index 0000000..5e89eea
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Supervisor.java
@@ -0,0 +1,357 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.storm.Config;
+import org.apache.storm.StormTimer;
+import org.apache.storm.cluster.ClusterStateContext;
+import org.apache.storm.cluster.ClusterUtils;
+import org.apache.storm.cluster.DaemonType;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.daemon.DaemonCommon;
+import org.apache.storm.daemon.supervisor.timer.SupervisorHealthCheck;
+import org.apache.storm.daemon.supervisor.timer.SupervisorHeartbeat;
+import org.apache.storm.daemon.supervisor.timer.UpdateBlobs;
+import org.apache.storm.event.EventManager;
+import org.apache.storm.event.EventManagerImp;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.localizer.AsyncLocalizer;
+import org.apache.storm.localizer.ILocalizer;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.messaging.IContext;
+import org.apache.storm.metric.StormMetricsRegistry;
+import org.apache.storm.scheduler.ISupervisor;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.apache.storm.utils.VersionInfo;
+import org.apache.zookeeper.data.ACL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Supervisor implements DaemonCommon, AutoCloseable {
+    private static final Logger LOG = LoggerFactory.getLogger(Supervisor.class);
+    private final Map<String, Object> conf;
+    private final IContext sharedContext;
+    private volatile boolean active;
+    private final ISupervisor iSupervisor;
+    private final Utils.UptimeComputer upTime;
+    private final String stormVersion;
+    private final IStormClusterState stormClusterState;
+    private final LocalState localState;
+    private final String supervisorId;
+    private final String assignmentId;
+    private final String hostName;
+    // used for reporting used ports when heartbeating
+    private final AtomicReference<Map<Long, LocalAssignment>> currAssignment;
+    private final StormTimer heartbeatTimer;
+    private final StormTimer eventTimer;
+    private final StormTimer blobUpdateTimer;
+    private final Localizer localizer;
+    private final ILocalizer asyncLocalizer;
+    private EventManager eventManager;
+    private ReadClusterState readState;
+    
+    private Supervisor(ISupervisor iSupervisor) throws IOException {
+        this(Utils.readStormConfig(), null, iSupervisor);
+    }
+    
+    public Supervisor(Map<String, Object> conf, IContext sharedContext, ISupervisor iSupervisor) throws IOException {
+        this.conf = conf;
+        this.iSupervisor = iSupervisor;
+        this.active = true;
+        this.upTime = Utils.makeUptimeComputer();
+        this.stormVersion = VersionInfo.getVersion();
+        this.sharedContext = sharedContext;
+        
+        iSupervisor.prepare(conf, ConfigUtils.supervisorIsupervisorDir(conf));
+        
+        List<ACL> acls = null;
+        if (Utils.isZkAuthenticationConfiguredStormServer(conf)) {
+            acls = SupervisorUtils.supervisorZkAcls();
+        }
+
+        try {
+            this.stormClusterState = ClusterUtils.mkStormClusterState(conf, acls, new ClusterStateContext(DaemonType.SUPERVISOR));
+        } catch (Exception e) {
+            LOG.error("supervisor can't create stormClusterState");
+            throw Utils.wrapInRuntime(e);
+        }
+
+        try {
+            this.localState = ConfigUtils.supervisorState(conf);
+            this.localizer = Utils.createLocalizer(conf, ConfigUtils.supervisorLocalDir(conf));
+            this.asyncLocalizer = new AsyncLocalizer(conf, this.localizer);
+        } catch (IOException e) {
+            throw Utils.wrapInRuntime(e);
+        }
+        this.supervisorId = iSupervisor.getSupervisorId();
+        this.assignmentId = iSupervisor.getAssignmentId();
+
+        try {
+            this.hostName = Utils.hostname(conf);
+        } catch (UnknownHostException e) {
+            throw Utils.wrapInRuntime(e);
+        }
+
+        this.currAssignment = new AtomicReference<Map<Long, LocalAssignment>>(new HashMap<Long,LocalAssignment>());
+
+        this.heartbeatTimer = new StormTimer(null, new DefaultUncaughtExceptionHandler());
+
+        this.eventTimer = new StormTimer(null, new DefaultUncaughtExceptionHandler());
+
+        this.blobUpdateTimer = new StormTimer("blob-update-timer", new DefaultUncaughtExceptionHandler());
+    }
+    
+    public String getId() {
+        return supervisorId;
+    }
+    
+    IContext getSharedContext() {
+        return sharedContext;
+    }
+
+    public Map<String, Object> getConf() {
+        return conf;
+    }
+
+    public ISupervisor getiSupervisor() {
+        return iSupervisor;
+    }
+
+    public Utils.UptimeComputer getUpTime() {
+        return upTime;
+    }
+
+    public String getStormVersion() {
+        return stormVersion;
+    }
+
+    public IStormClusterState getStormClusterState() {
+        return stormClusterState;
+    }
+
+    LocalState getLocalState() {
+        return localState;
+    }
+
+    public String getAssignmentId() {
+        return assignmentId;
+    }
+
+    public String getHostName() {
+        return hostName;
+    }
+
+    public AtomicReference<Map<Long, LocalAssignment>> getCurrAssignment() {
+        return currAssignment;
+    }
+
+    public Localizer getLocalizer() {
+        return localizer;
+    }
+    
+    ILocalizer getAsyncLocalizer() {
+        return asyncLocalizer;
+    }
+    
+    EventManager getEventManger() {
+        return eventManager;
+    }
+    
+    /**
+     * Launch the supervisor
+     */
+    public void launch() throws Exception {
+        LOG.info("Starting Supervisor with conf {}", conf);
+        String path = ConfigUtils.supervisorTmpDir(conf);
+        FileUtils.cleanDirectory(new File(path));
+
+        Localizer localizer = getLocalizer();
+
+        SupervisorHeartbeat hb = new SupervisorHeartbeat(conf, this);
+        hb.run();
+        // should synchronize supervisor so it doesn't launch anything after being down (optimization)
+        Integer heartbeatFrequency = Utils.getInt(conf.get(Config.SUPERVISOR_HEARTBEAT_FREQUENCY_SECS));
+        heartbeatTimer.scheduleRecurring(0, heartbeatFrequency, hb);
+
+        this.eventManager = new EventManagerImp(false);
+        this.readState = new ReadClusterState(this);
+        
+        Set<String> downloadedTopoIds = SupervisorUtils.readDownloadedTopologyIds(conf);
+        for (String topoId : downloadedTopoIds) {
+            SupervisorUtils.addBlobReferences(localizer, topoId, conf);
+        }
+        // do this after adding the references so we don't try to clean things being used
+        localizer.startCleaner();
+
+        UpdateBlobs updateBlobsThread = new UpdateBlobs(this);
+
+        if ((Boolean) conf.get(Config.SUPERVISOR_ENABLE)) {
+            // This isn't strictly necessary, but it doesn't hurt and ensures that the machine stays up
+            // to date even if callbacks don't all work exactly right
+            eventTimer.scheduleRecurring(0, 10, new EventManagerPushCallback(readState, eventManager));
+
+            // Blob update thread. Starts with 30 seconds delay, every 30 seconds
+            blobUpdateTimer.scheduleRecurring(30, 30, new EventManagerPushCallback(updateBlobsThread, eventManager));
+
+            // supervisor health check
+            eventTimer.scheduleRecurring(300, 300, new SupervisorHealthCheck(this));
+        }
+        LOG.info("Starting supervisor with id {} at host {}.", getId(), getHostName());
+    }
+
+    /**
+     * start distribute supervisor
+     */
+    private void launchDaemon() {
+        LOG.info("Starting supervisor for storm version '{}'.", VersionInfo.getVersion());
+        try {
+            Map<String, Object> conf = getConf();
+            if (ConfigUtils.isLocalMode(conf)) {
+                throw new IllegalArgumentException("Cannot start server in local mode!");
+            }
+            launch();
+            Utils.addShutdownHookWithForceKillIn1Sec(() -> {this.close();});
+            registerWorkerNumGauge("supervisor:num-slots-used-gauge", conf);
+            StormMetricsRegistry.startMetricsReporters(conf);
+        } catch (Exception e) {
+            LOG.error("Failed to start supervisor\n", e);
+            System.exit(1);
+        }
+    }
+
+    private void registerWorkerNumGauge(String name, final Map<String, Object> conf) {
+        StormMetricsRegistry.registerGauge(name, new Callable<Integer>() {
+            @Override
+            public Integer call() throws Exception {
+                Collection<String> pids = SupervisorUtils.supervisorWorkerIds(conf);
+                return pids.size();
+            }
+        });
+    }
+    
+    @Override
+    public void close() {
+        try {
+            LOG.info("Shutting down supervisor {}", getId());
+            this.active = false;
+            heartbeatTimer.close();
+            eventTimer.close();
+            blobUpdateTimer.close();
+            if (eventManager != null) {
+                eventManager.close();
+            }
+            if (readState != null) {
+                readState.close();
+            }
+            getStormClusterState().disconnect();
+        } catch (Exception e) {
+            LOG.error("Error Shutting down", e);
+        }
+    }
+    
+    void killWorkers(Collection<String> workerIds, ContainerLauncher launcher) throws InterruptedException, IOException {
+        HashSet<Killable> containers = new HashSet<>();
+        for (String workerId : workerIds) {
+            try {
+                Killable k = launcher.recoverContainer(workerId, localState);
+                if (!k.areAllProcessesDead()) {
+                    k.kill();
+                    containers.add(k);
+                } else {
+                    k.cleanUp();
+                }
+            } catch (Exception e) {
+                LOG.error("Error trying to kill {}", workerId, e);
+            }
+        }
+        int shutdownSleepSecs = Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_SHUTDOWN_SLEEP_SECS), 1);
+        if (!containers.isEmpty()) {
+            Time.sleepSecs(shutdownSleepSecs);
+        }
+        for (Killable k: containers) {
+            try {
+                k.forceKill();
+                long start = Time.currentTimeMillis();
+                while(!k.areAllProcessesDead()) {
+                    if ((Time.currentTimeMillis() - start) > 10_000) {
+                        throw new RuntimeException("Giving up on killing " + k 
+                                + " after " + (Time.currentTimeMillis() - start) + " ms");
+                    }
+                    Time.sleep(100);
+                    k.forceKill();
+                }
+                k.cleanUp();
+            } catch (Exception e) {
+                LOG.error("Error trying to clean up {}", k, e);
+            }
+        }
+    }
+
+    public void shutdownAllWorkers() {
+        if (readState != null) {
+            readState.shutdownAllWorkers();
+        } else {
+            try {
+                ContainerLauncher launcher = ContainerLauncher.make(getConf(), getId(), getSharedContext());
+                killWorkers(SupervisorUtils.supervisorWorkerIds(conf), launcher);
+            } catch (Exception e) {
+                throw Utils.wrapInRuntime(e);
+            }
+        }
+    }
+
+    @Override
+    public boolean isWaiting() {
+        if (!active) {
+            return true;
+        }
+
+        if (heartbeatTimer.isTimerWaiting() && eventTimer.isTimerWaiting() && eventManager.waiting()) {
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * supervisor daemon enter entrance
+     *
+     * @param args
+     */
+    public static void main(String[] args) throws Exception {
+        Utils.setupDefaultUncaughtExceptionHandler();
+        @SuppressWarnings("resource")
+        Supervisor instance = new Supervisor(new StandaloneSupervisor());
+        instance.launchDaemon();
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
new file mode 100644
index 0000000..0784631
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/SupervisorUtils.java
@@ -0,0 +1,311 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.Config;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.localizer.LocalResource;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+public class SupervisorUtils {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SupervisorUtils.class);
+
+    private static final SupervisorUtils INSTANCE = new SupervisorUtils();
+    private static SupervisorUtils _instance = INSTANCE;
+    public static void setInstance(SupervisorUtils u) {
+        _instance = u;
+    }
+    public static void resetInstance() {
+        _instance = INSTANCE;
+    }
+
+    static Process processLauncher(Map<String, Object> conf, String user, List<String> commandPrefix, List<String> args, Map<String, String> environment, final String logPreFix,
+                                          final ExitCodeCallback exitCodeCallback, File dir) throws IOException {
+        if (StringUtils.isBlank(user)) {
+            throw new IllegalArgumentException("User cannot be blank when calling processLauncher.");
+        }
+        String wlinitial = (String) (conf.get(Config.SUPERVISOR_WORKER_LAUNCHER));
+        String stormHome = ConfigUtils.concatIfNotNull(System.getProperty("storm.home"));
+        String wl;
+        if (StringUtils.isNotBlank(wlinitial)) {
+            wl = wlinitial;
+        } else {
+            wl = stormHome + "/bin/worker-launcher";
+        }
+        List<String> commands = new ArrayList<>();
+        if (commandPrefix != null){
+            commands.addAll(commandPrefix);
+        }
+        commands.add(wl);
+        commands.add(user);
+        commands.addAll(args);
+        LOG.info("Running as user: {} command: {}", user, commands);
+        return SupervisorUtils.launchProcess(commands, environment, logPreFix, exitCodeCallback, dir);
+    }
+
+    public static int processLauncherAndWait(Map<String, Object> conf, String user, List<String> args, final Map<String, String> environment, final String logPreFix)
+            throws IOException {
+        int ret = 0;
+        Process process = processLauncher(conf, user, null, args, environment, logPreFix, null, null);
+        if (StringUtils.isNotBlank(logPreFix))
+            Utils.readAndLogStream(logPreFix, process.getInputStream());
+        try {
+            process.waitFor();
+        } catch (InterruptedException e) {
+            LOG.info("{} interrupted.", logPreFix);
+        }
+        ret = process.exitValue();
+        return ret;
+    }
+
+    public static void setupStormCodeDir(Map<String, Object> conf, Map<String, Object> stormConf, String dir) throws IOException {
+        if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
+            String logPrefix = "setup conf for " + dir;
+            List<String> commands = new ArrayList<>();
+            commands.add("code-dir");
+            commands.add(dir);
+            processLauncherAndWait(conf, (String) (stormConf.get(Config.TOPOLOGY_SUBMITTER_USER)), commands, null, logPrefix);
+        }
+    }
+
+    public static void rmrAsUser(Map<String, Object> conf, String id, String path) throws IOException {
+        String user = Utils.getFileOwner(path);
+        String logPreFix = "rmr " + id;
+        List<String> commands = new ArrayList<>();
+        commands.add("rmr");
+        commands.add(path);
+        SupervisorUtils.processLauncherAndWait(conf, user, commands, null, logPreFix);
+        if (Utils.checkFileExists(path)) {
+            throw new RuntimeException(path + " was not deleted.");
+        }
+    }
+
+    /**
+     * Given the blob information returns the value of the uncompress field, handling it either being a string or a boolean value, or if it's not specified then
+     * returns false
+     * 
+     * @param blobInfo
+     * @return
+     */
+    public static Boolean shouldUncompressBlob(Map<String, Object> blobInfo) {
+        return Utils.getBoolean(blobInfo.get("uncompress"), false);
+    }
+
+    /**
+     * Returns a list of LocalResources based on the blobstore-map passed in
+     * 
+     * @param blobstoreMap
+     * @return
+     */
+    public static List<LocalResource> blobstoreMapToLocalresources(Map<String, Map<String, Object>> blobstoreMap) {
+        List<LocalResource> localResourceList = new ArrayList<>();
+        if (blobstoreMap != null) {
+            for (Map.Entry<String, Map<String, Object>> map : blobstoreMap.entrySet()) {
+                LocalResource localResource = new LocalResource(map.getKey(), shouldUncompressBlob(map.getValue()));
+                localResourceList.add(localResource);
+            }
+        }
+        return localResourceList;
+    }
+
+    /**
+     * For each of the downloaded topologies, adds references to the blobs that the topologies are using. This is used to reconstruct the cache on restart.
+     * 
+     * @param localizer
+     * @param stormId
+     * @param conf
+     */
+    static void addBlobReferences(Localizer localizer, String stormId, Map<String, Object> conf) throws IOException {
+        Map<String, Object> stormConf = ConfigUtils.readSupervisorStormConf(conf, stormId);
+        Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) stormConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
+        String user = (String) stormConf.get(Config.TOPOLOGY_SUBMITTER_USER);
+        String topoName = (String) stormConf.get(Config.TOPOLOGY_NAME);
+        List<LocalResource> localresources = SupervisorUtils.blobstoreMapToLocalresources(blobstoreMap);
+        if (blobstoreMap != null) {
+            localizer.addReferences(localresources, user, topoName);
+        }
+    }
+
+    public static Set<String> readDownloadedTopologyIds(Map<String, Object> conf) throws IOException {
+        Set<String> stormIds = new HashSet<>();
+        String path = ConfigUtils.supervisorStormDistRoot(conf);
+        Collection<String> rets = Utils.readDirContents(path);
+        for (String ret : rets) {
+            stormIds.add(URLDecoder.decode(ret));
+        }
+        return stormIds;
+    }
+
+    public static Collection<String> supervisorWorkerIds(Map<String, Object> conf) {
+        String workerRoot = ConfigUtils.workerRoot(conf);
+        return Utils.readDirContents(workerRoot);
+    }
+
+    static boolean doRequiredTopoFilesExist(Map<String, Object> conf, String stormId) throws IOException {
+        String stormroot = ConfigUtils.supervisorStormDistRoot(conf, stormId);
+        String stormjarpath = ConfigUtils.supervisorStormJarPath(stormroot);
+        String stormcodepath = ConfigUtils.supervisorStormCodePath(stormroot);
+        String stormconfpath = ConfigUtils.supervisorStormConfPath(stormroot);
+        if (!Utils.checkFileExists(stormroot))
+            return false;
+        if (!Utils.checkFileExists(stormcodepath))
+            return false;
+        if (!Utils.checkFileExists(stormconfpath))
+            return false;
+        if (ConfigUtils.isLocalMode(conf) || Utils.checkFileExists(stormjarpath))
+            return true;
+        return false;
+    }
+
+    /**
+     * map from worker id to heartbeat
+     *
+     * @param conf
+     * @return
+     * @throws Exception
+     */
+    public static Map<String, LSWorkerHeartbeat> readWorkerHeartbeats(Map<String, Object> conf) throws Exception {
+        return _instance.readWorkerHeartbeatsImpl(conf);
+    }
+
+    public Map<String, LSWorkerHeartbeat> readWorkerHeartbeatsImpl(Map<String, Object> conf) throws Exception {
+        Map<String, LSWorkerHeartbeat> workerHeartbeats = new HashMap<>();
+
+        Collection<String> workerIds = SupervisorUtils.supervisorWorkerIds(conf);
+
+        for (String workerId : workerIds) {
+            LSWorkerHeartbeat whb = readWorkerHeartbeat(conf, workerId);
+            // ATTENTION: whb can be null
+            workerHeartbeats.put(workerId, whb);
+        }
+        return workerHeartbeats;
+    }
+
+
+    /**
+     * get worker heartbeat by workerId
+     *
+     * @param conf
+     * @param workerId
+     * @return
+     * @throws IOException
+     */
+    private static LSWorkerHeartbeat readWorkerHeartbeat(Map<String, Object> conf, String workerId) {
+        return _instance.readWorkerHeartbeatImpl(conf, workerId);
+    }
+
+    protected LSWorkerHeartbeat readWorkerHeartbeatImpl(Map<String, Object> conf, String workerId) {
+        try {
+            LocalState localState = ConfigUtils.workerState(conf, workerId);
+            return localState.getWorkerHeartBeat();
+        } catch (Exception e) {
+            LOG.warn("Failed to read local heartbeat for workerId : {},Ignoring exception.", workerId, e);
+            return null;
+        }
+    }
+
+    public static boolean  isWorkerHbTimedOut(int now, LSWorkerHeartbeat whb, Map<String, Object> conf) {
+        return _instance.isWorkerHbTimedOutImpl(now, whb, conf);
+    }
+
+    private  boolean  isWorkerHbTimedOutImpl(int now, LSWorkerHeartbeat whb, Map<String, Object> conf) {
+        return (now - whb.get_time_secs()) > Utils.getInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS));
+    }
+    
+    /**
+     * Launch a new process as per {@link java.lang.ProcessBuilder} with a given
+     * callback.
+     * @param command the command to be executed in the new process
+     * @param environment the environment to be applied to the process. Can be
+     *                    null.
+     * @param logPrefix a prefix for log entries from the output of the process.
+     *                  Can be null.
+     * @param exitCodeCallback code to be called passing the exit code value
+     *                         when the process completes
+     * @param dir the working directory of the new process
+     * @return the new process
+     * @throws IOException
+     * @see java.lang.ProcessBuilder
+     */
+    public static Process launchProcess(List<String> command,
+                                        Map<String,String> environment,
+                                        final String logPrefix,
+                                        final ExitCodeCallback exitCodeCallback,
+                                        File dir)
+            throws IOException {
+        ProcessBuilder builder = new ProcessBuilder(command);
+        Map<String,String> procEnv = builder.environment();
+        if (dir != null) {
+            builder.directory(dir);
+        }
+        builder.redirectErrorStream(true);
+        if (environment != null) {
+            procEnv.putAll(environment);
+        }
+        final Process process = builder.start();
+        if (logPrefix != null || exitCodeCallback != null) {
+            Utils.asyncLoop(new Callable<Object>() {
+                public Object call() {
+                    if (logPrefix != null ) {
+                        Utils.readAndLogStream(logPrefix,
+                                process.getInputStream());
+                    }
+                    if (exitCodeCallback != null) {
+                        try {
+                            process.waitFor();
+                            exitCodeCallback.call(process.exitValue());
+                        } catch (InterruptedException ie) {
+                            LOG.info("{} interrupted", logPrefix);
+                            exitCodeCallback.call(-1);
+                        }
+                    }
+                    return null; // Run only once.
+                }
+            });
+        }
+        return process;
+    }
+    
+    static List<ACL> supervisorZkAcls() {
+        final List<ACL> acls = new ArrayList<>();
+        acls.add(ZooDefs.Ids.CREATOR_ALL_ACL.get(0));
+        acls.add(new ACL((ZooDefs.Perms.READ ^ ZooDefs.Perms.CREATE), ZooDefs.Ids.ANYONE_ID_UNSAFE));
+        return acls;
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java
new file mode 100644
index 0000000..0017092
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHealthCheck.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.daemon.supervisor.timer;
+
+import java.util.Map;
+
+import org.apache.storm.command.HealthCheck;
+import org.apache.storm.daemon.supervisor.Supervisor;
+
+public class SupervisorHealthCheck implements Runnable {
+    private final Supervisor supervisor;
+
+    public SupervisorHealthCheck(Supervisor supervisor) {
+        this.supervisor = supervisor;
+    }
+
+    @Override
+    public void run() {
+        Map<String, Object> conf = supervisor.getConf();
+        int healthCode = HealthCheck.healthCheck(conf);
+        if (healthCode != 0) {
+            supervisor.shutdownAllWorkers();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
new file mode 100644
index 0000000..34c5682
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor.timer;
+
+import org.apache.storm.Config;
+import org.apache.storm.cluster.IStormClusterState;
+import org.apache.storm.daemon.supervisor.Supervisor;
+import org.apache.storm.generated.SupervisorInfo;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SupervisorHeartbeat implements Runnable {
+
+     private final IStormClusterState stormClusterState;
+     private final String supervisorId;
+     private final Map<String, Object> conf;
+     private final Supervisor supervisor;
+
+    public SupervisorHeartbeat(Map<String, Object> conf, Supervisor supervisor) {
+        this.stormClusterState = supervisor.getStormClusterState();
+        this.supervisorId = supervisor.getId();
+        this.supervisor = supervisor;
+        this.conf = conf;
+    }
+
+    private SupervisorInfo buildSupervisorInfo(Map<String, Object> conf, Supervisor supervisor) {
+        SupervisorInfo supervisorInfo = new SupervisorInfo();
+        supervisorInfo.set_time_secs(Time.currentTimeSecs());
+        supervisorInfo.set_hostname(supervisor.getHostName());
+        supervisorInfo.set_assignment_id(supervisor.getAssignmentId());
+
+        List<Long> usedPorts = new ArrayList<>();
+        usedPorts.addAll(supervisor.getCurrAssignment().get().keySet());
+        supervisorInfo.set_used_ports(usedPorts);
+        List metaDatas = (List)supervisor.getiSupervisor().getMetadata();
+        List<Long> portList = new ArrayList<>();
+        if (metaDatas != null){
+            for (Object data : metaDatas){
+                Integer port = Utils.getInt(data);
+                if (port != null)
+                    portList.add(port.longValue());
+            }
+        }
+
+        supervisorInfo.set_meta(portList);
+        supervisorInfo.set_scheduler_meta((Map<String, String>) conf.get(Config.SUPERVISOR_SCHEDULER_META));
+        supervisorInfo.set_uptime_secs(supervisor.getUpTime().upTime());
+        supervisorInfo.set_version(supervisor.getStormVersion());
+        supervisorInfo.set_resources_map(mkSupervisorCapacities(conf));
+        return supervisorInfo;
+    }
+
+    private Map<String, Double> mkSupervisorCapacities(Map conf) {
+        Map<String, Double> ret = new HashMap<String, Double>();
+        Double mem = (double) (conf.get(Config.SUPERVISOR_MEMORY_CAPACITY_MB));
+        ret.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, mem);
+        Double cpu = (double) (conf.get(Config.SUPERVISOR_CPU_CAPACITY));
+        ret.put(Config.SUPERVISOR_CPU_CAPACITY, cpu);
+        return ret;
+    }
+
+    @Override
+    public void run() {
+        SupervisorInfo supervisorInfo = buildSupervisorInfo(conf, supervisor);
+        stormClusterState.supervisorHeartbeat(supervisorId, supervisorInfo);
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/UpdateBlobs.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/UpdateBlobs.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/UpdateBlobs.java
new file mode 100644
index 0000000..0b6d996
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/timer/UpdateBlobs.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor.timer;
+
+import org.apache.storm.Config;
+import org.apache.storm.daemon.supervisor.Supervisor;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.KeyNotFoundException;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.localizer.LocalResource;
+import org.apache.storm.localizer.Localizer;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.NimbusLeaderNotFoundException;
+import org.apache.storm.utils.Utils;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * downloads all blobs listed in the topology configuration for all topologies assigned to this supervisor, and creates version files with a suffix. The
+ * Runnable is intended to be run periodically by a timer, created elsewhere.
+ */
+public class UpdateBlobs implements Runnable {
+
+    private static final Logger LOG = LoggerFactory.getLogger(UpdateBlobs.class);
+
+    private Supervisor supervisor;
+
+    public UpdateBlobs(Supervisor supervisor) {
+        this.supervisor = supervisor;
+    }
+
+    @Override
+    public void run() {
+        try {
+            Map<String, Object> conf = supervisor.getConf();
+            Set<String> downloadedStormIds = SupervisorUtils.readDownloadedTopologyIds(conf);
+            AtomicReference<Map<Long, LocalAssignment>> newAssignment = supervisor.getCurrAssignment();
+            Set<String> assignedStormIds = new HashSet<>();
+            for (LocalAssignment localAssignment : newAssignment.get().values()) {
+                assignedStormIds.add(localAssignment.get_topology_id());
+            }
+            for (String stormId : downloadedStormIds) {
+                if (assignedStormIds.contains(stormId)) {
+                    String stormRoot = ConfigUtils.supervisorStormDistRoot(conf, stormId);
+                    LOG.debug("Checking Blob updates for storm topology id {} With target_dir: {}", stormId, stormRoot);
+                    updateBlobsForTopology(conf, stormId, supervisor.getLocalizer());
+                }
+            }
+        } catch (Exception e) {
+            if (Utils.exceptionCauseIsInstanceOf(TTransportException.class, e)) {
+                LOG.error("Network error while updating blobs, will retry again later", e);
+            } else if (Utils.exceptionCauseIsInstanceOf(NimbusLeaderNotFoundException.class, e)) {
+                LOG.error("Nimbus unavailable to update blobs, will retry again later", e);
+            } else {
+                throw Utils.wrapInRuntime(e);
+            }
+        }
+    }
+
+    /**
+     * Update each blob listed in the topology configuration if the latest version of the blob has not been downloaded.
+     * 
+     * @param conf
+     * @param stormId
+     * @param localizer
+     * @throws IOException
+     */
+    private void updateBlobsForTopology(Map conf, String stormId, Localizer localizer) throws IOException {
+        Map stormConf = ConfigUtils.readSupervisorStormConf(conf, stormId);
+        Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) stormConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
+        String user = (String) stormConf.get(Config.TOPOLOGY_SUBMITTER_USER);
+        List<LocalResource> localresources = SupervisorUtils.blobstoreMapToLocalresources(blobstoreMap);
+        try {
+            localizer.updateBlobs(localresources, user);
+        } catch (AuthorizationException authExp) {
+            LOG.error("AuthorizationException error", authExp);
+        } catch (KeyNotFoundException knf) {
+            LOG.error("KeyNotFoundException error", knf);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/event/EventManager.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/event/EventManager.java b/storm-core/src/jvm/org/apache/storm/event/EventManager.java
new file mode 100644
index 0000000..64536c1
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/event/EventManager.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.event;
+
+public interface EventManager extends AutoCloseable {
+    void add(Runnable eventFn);
+
+    boolean waiting();
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/event/EventManagerImp.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/event/EventManagerImp.java b/storm-core/src/jvm/org/apache/storm/event/EventManagerImp.java
new file mode 100644
index 0000000..42e6d6b
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/event/EventManagerImp.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.event;
+
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InterruptedIOException;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class EventManagerImp implements EventManager {
+    private static final Logger LOG = LoggerFactory.getLogger(EventManagerImp.class);
+
+    private AtomicInteger added;
+    private AtomicInteger processed;
+    private AtomicBoolean running;
+    private Thread runner;
+
+    private LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>();
+
+    public EventManagerImp(boolean isDaemon) {
+        added = new AtomicInteger();
+        processed = new AtomicInteger();
+        running = new AtomicBoolean(true);
+        runner = new Thread() {
+            @Override
+            public void run() {
+                while (running.get()) {
+                    try {
+                        Runnable r = queue.take();
+                        if (r == null) {
+                            return;
+                        }
+
+                        r.run();
+                        proccessInc();
+                    } catch (Throwable t) {
+                        if (Utils.exceptionCauseIsInstanceOf(InterruptedIOException.class, t)) {
+                            LOG.info("Event manager interrupted while doing IO");
+                        } else if (Utils.exceptionCauseIsInstanceOf(InterruptedException.class, t)) {
+                            LOG.info("Event manager interrupted");
+                        } else {
+                            LOG.error("{} Error when processing event", t);
+                            Utils.exitProcess(20, "Error when processing an event");
+                        }
+                    }
+                }
+            }
+        };
+        runner.setDaemon(isDaemon);
+        runner.start();
+    }
+
+    public void proccessInc() {
+        processed.incrementAndGet();
+    }
+
+    @Override
+    public void add(Runnable eventFn) {
+        if (!running.get()) {
+            throw new RuntimeException("Cannot add events to a shutdown event manager");
+        }
+        added.incrementAndGet();
+        queue.add(eventFn);
+    }
+
+    @Override
+    public boolean waiting() {
+        return (Time.isThreadWaiting(runner) || (processed.get() == added.get()));
+    }
+
+    @Override
+    public void close() throws Exception {
+        running.set(false);
+        runner.interrupt();
+        runner.join();
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java b/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
new file mode 100644
index 0000000..7887281
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
@@ -0,0 +1,432 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.localizer;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.JarURLConnection;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.storm.Config;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.daemon.Shutdownable;
+import org.apache.storm.daemon.supervisor.AdvancedFSOps;
+import org.apache.storm.daemon.supervisor.SupervisorUtils;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * This is a wrapper around the Localizer class that provides the desired
+ * async interface to Slot.
+ */
+public class AsyncLocalizer implements ILocalizer, Shutdownable {
+    /**
+     * A future that has already completed.
+     */
+    private static class AllDoneFuture implements Future<Void> {
+
+        @Override
+        public boolean cancel(boolean mayInterruptIfRunning) {
+            return false;
+        }
+
+        @Override
+        public boolean isCancelled() {
+            return false;
+        }
+
+        @Override
+        public boolean isDone() {
+            return true;
+        }
+
+        @Override
+        public Void get() {
+            return null;
+        }
+
+        @Override
+        public Void get(long timeout, TimeUnit unit) {
+            return null;
+        }
+
+    }
+
+    private static final Logger LOG = LoggerFactory.getLogger(AsyncLocalizer.class);
+
+    private final Localizer _localizer;
+    private final ExecutorService _execService;
+    private final boolean _isLocalMode;
+    private final Map<String, Object> _conf;
+    private final Map<String, LocalDownloadedResource> _basicPending;
+    private final Map<String, LocalDownloadedResource> _blobPending;
+    private final AdvancedFSOps _fsOps;
+
+    private class DownloadBaseBlobsDistributed implements Callable<Void> {
+        protected final String _topologyId;
+        protected final File _stormRoot;
+        
+        public DownloadBaseBlobsDistributed(String topologyId) throws IOException {
+            _topologyId = topologyId;
+            _stormRoot = new File(ConfigUtils.supervisorStormDistRoot(_conf, _topologyId));
+        }
+        
+        protected void downloadBaseBlobs(File tmproot) throws Exception {
+            String stormJarKey = ConfigUtils.masterStormJarKey(_topologyId);
+            String stormCodeKey = ConfigUtils.masterStormCodeKey(_topologyId);
+            String stormConfKey = ConfigUtils.masterStormConfKey(_topologyId);
+            String jarPath = ConfigUtils.supervisorStormJarPath(tmproot.getAbsolutePath());
+            String codePath = ConfigUtils.supervisorStormCodePath(tmproot.getAbsolutePath());
+            String confPath = ConfigUtils.supervisorStormConfPath(tmproot.getAbsolutePath());
+            _fsOps.forceMkdir(tmproot);
+            _fsOps.restrictDirectoryPermissions(tmproot);
+            ClientBlobStore blobStore = Utils.getClientBlobStoreForSupervisor(_conf);
+            try {
+                Utils.downloadResourcesAsSupervisor(stormJarKey, jarPath, blobStore);
+                Utils.downloadResourcesAsSupervisor(stormCodeKey, codePath, blobStore);
+                Utils.downloadResourcesAsSupervisor(stormConfKey, confPath, blobStore);
+            } finally {
+                blobStore.shutdown();
+            }
+            Utils.extractDirFromJar(jarPath, ConfigUtils.RESOURCES_SUBDIR, tmproot);
+        }
+        
+        @Override
+        public Void call() throws Exception {
+            try {
+                if (_fsOps.fileExists(_stormRoot)) {
+                    if (!_fsOps.supportsAtomicDirectoryMove()) {
+                        LOG.warn("{} may have partially downloaded blobs, recovering", _topologyId);
+                        _fsOps.deleteIfExists(_stormRoot);
+                    } else {
+                        LOG.warn("{} already downloaded blobs, skipping", _topologyId);
+                        return null;
+                    }
+                }
+                boolean deleteAll = true;
+                String tmproot = ConfigUtils.supervisorTmpDir(_conf) + Utils.FILE_PATH_SEPARATOR + Utils.uuid();
+                File tr = new File(tmproot);
+                try {
+                    downloadBaseBlobs(tr);
+                    _fsOps.moveDirectoryPreferAtomic(tr, _stormRoot);
+                    _fsOps.setupStormCodeDir(ConfigUtils.readSupervisorStormConf(_conf, _topologyId), _stormRoot);
+                    deleteAll = false;
+                } finally {
+                    if (deleteAll) {
+                        LOG.warn("Failed to download basic resources for topology-id {}", _topologyId);
+                        _fsOps.deleteIfExists(tr);
+                        _fsOps.deleteIfExists(_stormRoot);
+                    }
+                }
+                return null;
+            } catch (Exception e) {
+                LOG.warn("Caught Exception While Downloading (rethrowing)... ", e);
+                throw e;
+            }
+        }
+    }
+    
+    private class DownloadBaseBlobsLocal extends DownloadBaseBlobsDistributed {
+
+        public DownloadBaseBlobsLocal(String topologyId) throws IOException {
+            super(topologyId);
+        }
+        
+        @Override
+        protected void downloadBaseBlobs(File tmproot) throws Exception {
+            _fsOps.forceMkdir(tmproot);
+            String stormCodeKey = ConfigUtils.masterStormCodeKey(_topologyId);
+            String stormConfKey = ConfigUtils.masterStormConfKey(_topologyId);
+            File codePath = new File(ConfigUtils.supervisorStormCodePath(tmproot.getAbsolutePath()));
+            File confPath = new File(ConfigUtils.supervisorStormConfPath(tmproot.getAbsolutePath()));
+            BlobStore blobStore = Utils.getNimbusBlobStore(_conf, null);
+            try {
+                try (OutputStream codeOutStream = _fsOps.getOutputStream(codePath)){
+                    blobStore.readBlobTo(stormCodeKey, codeOutStream, null);
+                }
+                try (OutputStream confOutStream = _fsOps.getOutputStream(confPath)) {
+                    blobStore.readBlobTo(stormConfKey, confOutStream, null);
+                }
+            } finally {
+                blobStore.shutdown();
+            }
+
+            ClassLoader classloader = Thread.currentThread().getContextClassLoader();
+            String resourcesJar = AsyncLocalizer.resourcesJar();
+            URL url = classloader.getResource(ConfigUtils.RESOURCES_SUBDIR);
+
+            String targetDir = tmproot + Utils.FILE_PATH_SEPARATOR + ConfigUtils.RESOURCES_SUBDIR;
+
+            if (resourcesJar != null) {
+                LOG.info("Extracting resources from jar at {} to {}", resourcesJar, targetDir);
+                Utils.extractDirFromJar(resourcesJar, ConfigUtils.RESOURCES_SUBDIR, _stormRoot);
+            } else if (url != null) {
+                LOG.info("Copying resources at {} to {} ", url.toString(), targetDir);
+                if ("jar".equals(url.getProtocol())) {
+                    JarURLConnection urlConnection = (JarURLConnection) url.openConnection();
+                    Utils.extractDirFromJar(urlConnection.getJarFileURL().getFile(), ConfigUtils.RESOURCES_SUBDIR, _stormRoot);
+                } else {
+                    _fsOps.copyDirectory(new File(url.getFile()), new File(targetDir));
+                }
+            }
+        }
+    }
+    
+    private class DownloadBlobs implements Callable<Void> {
+        private final String _topologyId;
+
+        public DownloadBlobs(String topologyId) {
+            _topologyId = topologyId;
+        }
+
+        @Override
+        public Void call() throws Exception {
+            try {
+                String stormroot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId);
+                Map<String, Object> stormConf = ConfigUtils.readSupervisorStormConf(_conf, _topologyId);
+
+                @SuppressWarnings("unchecked")
+                Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) stormConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
+                String user = (String) stormConf.get(Config.TOPOLOGY_SUBMITTER_USER);
+                String topoName = (String) stormConf.get(Config.TOPOLOGY_NAME);
+
+                List<LocalResource> localResourceList = new ArrayList<>();
+                if (blobstoreMap != null) {
+                    List<LocalResource> tmp = SupervisorUtils.blobstoreMapToLocalresources(blobstoreMap);
+                    if (tmp != null) {
+                        localResourceList.addAll(tmp);
+                    }
+                }
+
+                StormTopology stormCode = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _fsOps);
+                List<String> dependencies = new ArrayList<>();
+                if (stormCode.is_set_dependency_jars()) {
+                    dependencies.addAll(stormCode.get_dependency_jars());
+                }
+                if (stormCode.is_set_dependency_artifacts()) {
+                    dependencies.addAll(stormCode.get_dependency_artifacts());
+                }
+                for (String dependency : dependencies) {
+                    localResourceList.add(new LocalResource(dependency, false));
+                }
+
+                if (!localResourceList.isEmpty()) {
+                    File userDir = _localizer.getLocalUserFileCacheDir(user);
+                    if (!_fsOps.fileExists(userDir)) {
+                        _fsOps.forceMkdir(userDir);
+                    }
+                    List<LocalizedResource> localizedResources = _localizer.getBlobs(localResourceList, user, topoName, userDir);
+                    _fsOps.setupBlobPermissions(userDir, user);
+                    for (LocalizedResource localizedResource : localizedResources) {
+                        String keyName = localizedResource.getKey();
+                        //The sym link we are pointing to
+                        File rsrcFilePath = new File(localizedResource.getCurrentSymlinkPath());
+
+                        String symlinkName = null;
+                        if (blobstoreMap != null) {
+                            Map<String, Object> blobInfo = blobstoreMap.get(keyName);
+                            if (blobInfo != null && blobInfo.containsKey("localname")) {
+                                symlinkName = (String) blobInfo.get("localname");
+                            } else {
+                                symlinkName = keyName;
+                            }
+                        } else {
+                            // all things are from dependencies
+                            symlinkName = keyName;
+                        }
+                        _fsOps.createSymlink(new File(stormroot, symlinkName), rsrcFilePath);
+                    }
+                }
+
+                return null;
+            } catch (Exception e) {
+                LOG.warn("Caught Exception While Downloading (rethrowing)... ", e);
+                throw e;
+            }
+        }
+    }
+    
+    //Visible for testing
+    AsyncLocalizer(Map<String, Object> conf, Localizer localizer, AdvancedFSOps ops) {
+        _conf = conf;
+        _isLocalMode = ConfigUtils.isLocalMode(conf);
+        _localizer = localizer;
+        _execService = Executors.newFixedThreadPool(1,  
+                new ThreadFactoryBuilder()
+                .setNameFormat("Async Localizer")
+                .build());
+        _basicPending = new HashMap<>();
+        _blobPending = new HashMap<>();
+        _fsOps = ops;
+    }
+    
+    public AsyncLocalizer(Map<String, Object> conf, Localizer localizer) {
+        this(conf, localizer, AdvancedFSOps.make(conf));
+    }
+
+    @Override
+    public synchronized Future<Void> requestDownloadBaseTopologyBlobs(final LocalAssignment assignment, final int port) throws IOException {
+        final String topologyId = assignment.get_topology_id();
+        LocalDownloadedResource localResource = _basicPending.get(topologyId);
+        if (localResource == null) {
+            Callable<Void> c;
+            if (_isLocalMode) {
+                c = new DownloadBaseBlobsLocal(topologyId);
+            } else {
+                c = new DownloadBaseBlobsDistributed(topologyId);
+            }
+            localResource = new LocalDownloadedResource(_execService.submit(c));
+            _basicPending.put(topologyId, localResource);
+        }
+        Future<Void> ret = localResource.reserve(port, assignment);
+        LOG.debug("Reserved basic {} {}", topologyId, localResource);
+        return ret;
+    }
+
+    private static String resourcesJar() throws IOException {
+        String path = Utils.currentClasspath();
+        if (path == null) {
+            return null;
+        }
+        
+        for (String jpath : path.split(File.pathSeparator)) {
+            if (jpath.endsWith(".jar")) {
+                if (Utils.zipDoesContainDir(jpath, ConfigUtils.RESOURCES_SUBDIR)) {
+                    return jpath;
+                }
+            }
+        }
+        return null;
+    }
+    
+    @Override
+    public synchronized void recoverRunningTopology(LocalAssignment assignment, int port) {
+        final String topologyId = assignment.get_topology_id();
+        LocalDownloadedResource localResource = _basicPending.get(topologyId);
+        if (localResource == null) {
+            localResource = new LocalDownloadedResource(new AllDoneFuture());
+            _basicPending.put(topologyId, localResource);
+        }
+        localResource.reserve(port, assignment);
+        LOG.debug("Recovered basic {} {}", topologyId, localResource);
+        
+        localResource = _blobPending.get(topologyId);
+        if (localResource == null) {
+            localResource = new LocalDownloadedResource(new AllDoneFuture());
+            _blobPending.put(topologyId, localResource);
+        }
+        localResource.reserve(port, assignment);
+        LOG.debug("Recovered blobs {} {}", topologyId, localResource);
+    }
+    
+    @Override
+    public synchronized Future<Void> requestDownloadTopologyBlobs(LocalAssignment assignment, int port) {
+        final String topologyId = assignment.get_topology_id();
+        LocalDownloadedResource localResource = _blobPending.get(topologyId);
+        if (localResource == null) {
+            Callable<Void> c = new DownloadBlobs(topologyId);
+            localResource = new LocalDownloadedResource(_execService.submit(c));
+            _blobPending.put(topologyId, localResource);
+        }
+        Future<Void> ret = localResource.reserve(port, assignment);
+        LOG.debug("Reserved blobs {} {}", topologyId, localResource);
+        return ret;
+    }
+
+    @Override
+    public synchronized void releaseSlotFor(LocalAssignment assignment, int port) throws IOException {
+        final String topologyId = assignment.get_topology_id();
+        LOG.debug("Releasing slot for {} {}", topologyId, port);
+        LocalDownloadedResource localResource = _blobPending.get(topologyId);
+        if (localResource == null || !localResource.release(port, assignment)) {
+            LOG.warn("Released blob reference {} {} for something that we didn't have {}", topologyId, port, localResource);
+        } else if (localResource.isDone()){
+            LOG.info("Released blob reference {} {} Cleaning up BLOB references...", topologyId, port);
+            _blobPending.remove(topologyId);
+            Map<String, Object> topoConf = ConfigUtils.readSupervisorStormConf(_conf, topologyId);
+            @SuppressWarnings("unchecked")
+            Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
+            if (blobstoreMap != null) {
+                String user = (String) topoConf.get(Config.TOPOLOGY_SUBMITTER_USER);
+                String topoName = (String) topoConf.get(Config.TOPOLOGY_NAME);
+                
+                for (Map.Entry<String, Map<String, Object>> entry : blobstoreMap.entrySet()) {
+                    String key = entry.getKey();
+                    Map<String, Object> blobInfo = entry.getValue();
+                    try {
+                        _localizer.removeBlobReference(key, user, topoName, SupervisorUtils.shouldUncompressBlob(blobInfo));
+                    } catch (Exception e) {
+                        throw new IOException(e);
+                    }
+                }
+            }
+        } else {
+            LOG.debug("Released blob reference {} {} still waiting on {}", topologyId, port, localResource);
+        }
+        
+        localResource = _basicPending.get(topologyId);
+        if (localResource == null || !localResource.release(port, assignment)) {
+            LOG.warn("Released basic reference {} {} for something that we didn't have {}", topologyId, port, localResource);
+        } else if (localResource.isDone()){
+            LOG.info("Released blob reference {} {} Cleaning up basic files...", topologyId, port);
+            _basicPending.remove(topologyId);
+            String path = ConfigUtils.supervisorStormDistRoot(_conf, topologyId);
+            _fsOps.deleteIfExists(new File(path), null, "rmr "+topologyId);
+        } else {
+            LOG.debug("Released basic reference {} {} still waiting on {}", topologyId, port, localResource);
+        }
+    }
+
+    @Override
+    public synchronized void cleanupUnusedTopologies() throws IOException {
+        File distRoot = new File(ConfigUtils.supervisorStormDistRoot(_conf));
+        LOG.info("Cleaning up unused topologies in {}", distRoot);
+        File[] children = distRoot.listFiles();
+        if (children != null) {
+            for (File topoDir : children) {
+                String topoId = URLDecoder.decode(topoDir.getName(), "UTF-8");
+                if (_basicPending.get(topoId) == null && _blobPending.get(topoId) == null) {
+                    _fsOps.deleteIfExists(topoDir, null, "rmr " + topoId);
+                }
+            }
+        }
+    }
+
+    @Override
+    public void shutdown() {
+        _execService.shutdown();
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/ILocalizer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/ILocalizer.java b/storm-core/src/jvm/org/apache/storm/localizer/ILocalizer.java
new file mode 100644
index 0000000..7105095
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/localizer/ILocalizer.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.localizer;
+
+import java.io.IOException;
+import java.util.concurrent.Future;
+
+import org.apache.storm.generated.LocalAssignment;
+
+/**
+ * Download blobs from the blob store and keep them up to date.
+ */
+public interface ILocalizer {
+
+    /**
+     * Recover a running topology by incrementing references for what it has already downloaded.
+     * @param assignment the assignment the resources are for
+     * @param port the port the topology is running in.
+     */
+    void recoverRunningTopology(LocalAssignment assignemnt, int port);
+    
+    /**
+     * Download storm.jar, storm.conf, and storm.ser for this topology if not done so already,
+     * and inc a reference count on them.
+     * @param assignment the assignment the resources are for
+     * @param port the port the topology is running on
+     * @return a future to let you know when they are done.
+     * @throws IOException on error 
+     */
+    Future<Void> requestDownloadBaseTopologyBlobs(LocalAssignment assignment, int port) throws IOException;
+
+    /**
+     * Download the blobs for this topology (reading in list in from the config)
+     * and inc reference count for these blobs.
+     * PRECONDITION: requestDownloadBaseTopologyBlobs has completed downloading.
+     * @param assignment the assignment the resources are for
+     * @param port the port the topology is running on
+     * @return a future to let you know when they are done.
+     */
+    Future<Void> requestDownloadTopologyBlobs(LocalAssignment assignment, int port);
+    
+    /**
+     * dec reference count on all blobs associated with this topology.
+     * @param assignment the assignment the resources are for
+     * @param port the port the topology is running on
+     * @throws IOException on any error
+     */
+    void releaseSlotFor(LocalAssignment assignment, int port) throws IOException;
+    
+    /**
+     * Clean up any topologies that are not in use right now.
+     * @throws IOException on any error.
+     */
+    void cleanupUnusedTopologies() throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/LocalDownloadedResource.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/LocalDownloadedResource.java b/storm-core/src/jvm/org/apache/storm/localizer/LocalDownloadedResource.java
new file mode 100644
index 0000000..570c951
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/localizer/LocalDownloadedResource.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.localizer;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.storm.generated.LocalAssignment;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LocalDownloadedResource {
+    private static final Logger LOG = LoggerFactory.getLogger(LocalDownloadedResource.class);
+    private static class NoCancelFuture<T> implements Future<T> {
+        private final Future<T> _wrapped;
+        
+        public NoCancelFuture(Future<T> wrapped) {
+            _wrapped = wrapped;
+        }
+        
+        @Override
+        public boolean cancel(boolean mayInterruptIfRunning) {
+            //cancel not currently supported
+            return false;
+        }
+
+        @Override
+        public boolean isCancelled() {
+            return false;
+        }
+
+        @Override
+        public boolean isDone() {
+            return _wrapped.isDone();
+        }
+
+        @Override
+        public T get() throws InterruptedException, ExecutionException {
+            return _wrapped.get();
+        }
+
+        @Override
+        public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+            return _wrapped.get(timeout, unit);
+        }
+    }
+    private static class PortNAssignment {
+        private final int _port;
+        private final LocalAssignment _assignment;
+        
+        public PortNAssignment(int port, LocalAssignment assignment) {
+            _port = port;
+            _assignment = assignment;
+        }
+        
+        @Override
+        public boolean equals(Object other) {
+            if (!(other instanceof PortNAssignment)) {
+                return false;
+            }
+            PortNAssignment pna = (PortNAssignment) other;
+            return pna._port == _port && _assignment.equals(pna._assignment); 
+        }
+        
+        @Override
+        public int hashCode() {
+            return (17 * _port) + _assignment.hashCode();
+        }
+        
+        @Override
+        public String toString() {
+            return "{"+ _port + " " + _assignment +"}";
+        }
+    }
+    private final Future<Void> _pending;
+    private final Set<PortNAssignment> _references;
+    private boolean _isDone;
+    
+    
+    public LocalDownloadedResource(Future<Void> pending) {
+        _pending = new NoCancelFuture<>(pending);
+        _references = new HashSet<>();
+        _isDone = false;
+    }
+
+    /**
+     * Reserve the resources
+     * @param port the port this is for
+     * @param la the assignment this is for
+     * @return a future that can be used to track it being downloaded.
+     */
+    public synchronized Future<Void> reserve(int port, LocalAssignment la) {
+        PortNAssignment pna = new PortNAssignment(port, la);
+        if (!_references.add(pna)) {
+            LOG.warn("Resources {} already reserved {} for this topology", pna, _references);
+        }
+        return _pending;
+    }
+    
+    /**
+     * Release a port from the reference count, and update isDone if all is done.
+     * @param port the port to release
+     * @param la the assignment to release
+     * @return true if the port was being counted else false
+     */
+    public synchronized boolean release(int port, LocalAssignment la) {
+        PortNAssignment pna = new PortNAssignment(port, la);
+        boolean ret = _references.remove(pna);
+        if (ret && _references.isEmpty()) {
+            _isDone = true;
+        }
+        return ret;
+    }
+    
+    /**
+     * Is this has been cleaned up completely.
+     * @return true if it is done else false
+     */
+    public synchronized boolean isDone() {
+        return _isDone;
+    }
+
+    @Override
+    public String toString() {
+        return _references.toString();
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSet.java b/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
index 380e777..9f42b47 100644
--- a/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
+++ b/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
@@ -93,7 +93,7 @@ public class LocalizedResourceRetentionSet {
           i.remove();
         } else {
           // since it failed to delete add it back so it gets retried
-          set.addResource(resource.getKey(), resource, resource.isUncompressed());
+          set.add(resource.getKey(), resource, resource.isUncompressed());
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceSet.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceSet.java b/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceSet.java
index b5f00c3..62d5b2f 100644
--- a/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceSet.java
+++ b/storm-core/src/jvm/org/apache/storm/localizer/LocalizedResourceSet.java
@@ -57,7 +57,7 @@ public class LocalizedResourceSet {
     return _localrsrcFiles.get(name);
   }
 
-  public void updateResource(String resourceName, LocalizedResource updatedResource,
+  public void putIfAbsent(String resourceName, LocalizedResource updatedResource,
                             boolean uncompress) {
     if (uncompress) {
       _localrsrcArchives.putIfAbsent(resourceName, updatedResource);
@@ -66,7 +66,7 @@ public class LocalizedResourceSet {
     }
   }
 
-  public void addResource(String resourceName, LocalizedResource newResource, boolean uncompress) {
+  public void add(String resourceName, LocalizedResource newResource, boolean uncompress) {
     if (uncompress) {
       _localrsrcArchives.put(resourceName, newResource);
     } else {
@@ -76,9 +76,9 @@ public class LocalizedResourceSet {
 
   public boolean exists(String resourceName, boolean uncompress) {
     if (uncompress) {
-      return (_localrsrcArchives.get(resourceName) != null);
+      return _localrsrcArchives.containsKey(resourceName);
     }
-    return (_localrsrcFiles.get(resourceName) != null);
+    return _localrsrcFiles.containsKey(resourceName);
   }
 
   public boolean remove(LocalizedResource resource) {

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/localizer/Localizer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/Localizer.java b/storm-core/src/jvm/org/apache/storm/localizer/Localizer.java
index b91cecb..0135397 100644
--- a/storm-core/src/jvm/org/apache/storm/localizer/Localizer.java
+++ b/storm-core/src/jvm/org/apache/storm/localizer/Localizer.java
@@ -63,20 +63,6 @@ import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
  */
 public class Localizer {
   public static final Logger LOG = LoggerFactory.getLogger(Localizer.class);
-
-  private Map _conf;
-  private int _threadPoolSize;
-  // thread pool for initial download
-  private ExecutorService _execService;
-  // thread pool for updates
-  private ExecutorService _updateExecService;
-  private int _blobDownloadRetries;
-
-  // track resources - user to resourceSet
-  private final ConcurrentMap<String, LocalizedResourceSet> _userRsrc = new
-      ConcurrentHashMap<String, LocalizedResourceSet>();
-
-  private String _localBaseDir;
   public static final String USERCACHE = "usercache";
   public static final String FILECACHE = "filecache";
 
@@ -85,13 +71,29 @@ public class Localizer {
   public static final String ARCHIVESDIR = "archives";
 
   private static final String TO_UNCOMPRESS = "_tmp_";
+  
+  
+  
+  private final Map<String, Object> _conf;
+  private final int _threadPoolSize;
+  // thread pool for initial download
+  private final ExecutorService _execService;
+  // thread pool for updates
+  private final ExecutorService _updateExecService;
+  private final int _blobDownloadRetries;
+
+  // track resources - user to resourceSet
+  private final ConcurrentMap<String, LocalizedResourceSet> _userRsrc = new
+      ConcurrentHashMap<String, LocalizedResourceSet>();
+
+  private final String _localBaseDir;
 
   // cleanup
   private long _cacheTargetSize;
   private long _cacheCleanupPeriod;
   private ScheduledExecutorService _cacheCleanupService;
 
-  public Localizer(Map conf, String baseDir) {
+  public Localizer(Map<String, Object> conf, String baseDir) {
     _conf = conf;
     _localBaseDir = baseDir;
     // default cache size 10GB, converted to Bytes
@@ -189,7 +191,7 @@ public class Localizer {
         LOG.debug("local file is: {} path is: {}", rsrc.getPath(), path);
         LocalizedResource lrsrc = new LocalizedResource(new File(path).getName(), path,
             uncompress);
-        lrsrcSet.addResource(lrsrc.getKey(), lrsrc, uncompress);
+        lrsrcSet.add(lrsrc.getKey(), lrsrc, uncompress);
       }
     }
   }
@@ -369,7 +371,7 @@ public class Localizer {
           if (newlrsrcSet == null) {
             newlrsrcSet = newSet;
           }
-          newlrsrcSet.updateResource(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
+          newlrsrcSet.putIfAbsent(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
           results.add(lrsrc);
         }
         catch (ExecutionException e) {
@@ -451,7 +453,7 @@ public class Localizer {
       for (Future<LocalizedResource> futureRsrc: futures) {
         LocalizedResource lrsrc = futureRsrc.get();
         lrsrc.addReference(topo);
-        lrsrcSet.addResource(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
+        lrsrcSet.add(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
         results.add(lrsrc);
       }
     } catch (ExecutionException e) {

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/metric/StormMetricsRegistry.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/metric/StormMetricsRegistry.java b/storm-core/src/jvm/org/apache/storm/metric/StormMetricsRegistry.java
new file mode 100644
index 0000000..d433920
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/metric/StormMetricsRegistry.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.metric;
+
+import com.codahale.metrics.*;
+
+import java.util.Map;
+import java.util.concurrent.Callable;
+
+import org.apache.storm.daemon.metrics.MetricsUtils;
+import org.apache.storm.daemon.metrics.reporters.PreparableReporter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("unchecked")
+public class StormMetricsRegistry {
+    private static final Logger LOG = LoggerFactory.getLogger(StormMetricsRegistry.class);
+    public static final MetricRegistry DEFAULT_REGISTRY = new MetricRegistry();
+
+    public static Meter registerMeter(String name) {
+        Meter meter = new Meter();
+        return register(name, meter);
+    }
+
+    // TODO: should replace Callable to Gauge<Integer> when nimbus.clj is translated to java
+    public static Gauge<Integer> registerGauge(final String name, final Callable fn) {
+        Gauge<Integer> gauge = new Gauge<Integer>() {
+            @Override
+            public Integer getValue() {
+                try {
+                    return (Integer) fn.call();
+                } catch (Exception e) {
+                    LOG.error("Error getting gauge value for {}", name, e);
+                }
+                return 0;
+            }
+        };
+        return register(name, gauge);
+    }
+
+    public static Histogram registerHistogram(String name, Reservoir reservoir) {
+        Histogram histogram = new Histogram(reservoir);
+        return register(name, histogram);
+    }
+
+    public static void startMetricsReporters(Map stormConf) {
+        for (PreparableReporter reporter : MetricsUtils.getPreparableReporters(stormConf)) {
+            startMetricsReporter(reporter, stormConf);
+        }
+    }
+
+    private static void startMetricsReporter(PreparableReporter reporter, Map stormConf) {
+        reporter.prepare(StormMetricsRegistry.DEFAULT_REGISTRY, stormConf);
+        reporter.start();
+        LOG.info("Started statistics report plugin...");
+    }
+
+    private static <T extends Metric> T register(String name, T metric) {
+        T ret;
+        try {
+            ret = DEFAULT_REGISTRY.register(name, metric);
+        } catch (IllegalArgumentException e) {
+            // swallow IllegalArgumentException when the metric exists already
+            ret = (T) DEFAULT_REGISTRY.getMetrics().get(name);
+            if (ret == null) {
+                throw e;
+            } else {
+                LOG.warn("Metric {} has already been registered", name);
+            }
+        }
+        return ret;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/nimbus/ILeaderElector.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/nimbus/ILeaderElector.java b/storm-core/src/jvm/org/apache/storm/nimbus/ILeaderElector.java
index 3c729ec..e8789df 100644
--- a/storm-core/src/jvm/org/apache/storm/nimbus/ILeaderElector.java
+++ b/storm-core/src/jvm/org/apache/storm/nimbus/ILeaderElector.java
@@ -37,14 +37,14 @@ public interface ILeaderElector extends Closeable {
      * check isLeader() to perform any leadership action. This method can be called
      * multiple times so it needs to be idempotent.
      */
-    void addToLeaderLockQueue();
+    void addToLeaderLockQueue() throws Exception;
 
     /**
      * Removes the caller from the leader lock queue. If the caller is leader
      * also releases the lock. This method can be called multiple times so it needs
      * to be idempotent.
      */
-    void removeFromLeaderLockQueue();
+    void removeFromLeaderLockQueue() throws Exception;
 
     /**
      *
@@ -62,7 +62,7 @@ public interface ILeaderElector extends Closeable {
      *
      * @return list of current nimbus addresses, includes leader.
      */
-    List<NimbusInfo> getAllNimbuses();
+    List<NimbusInfo> getAllNimbuses()throws Exception;
 
     /**
      * Method called to allow for cleanup. once close this object can not be reused.

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java b/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
index 32d28fe..8415ce3 100644
--- a/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
+++ b/storm-core/src/jvm/org/apache/storm/serialization/SerializationFactory.java
@@ -134,6 +134,21 @@ public class SerializationFactory {
         Map<String, Map<String, Integer>> streamNametoId = new HashMap<>();
         Map<String, Map<Integer, String>> streamIdToName = new HashMap<>();
 
+        /**
+         * "{:a 1  :b 2} -> {1 :a  2 :b}"
+         *
+         * Note: Only one key wins if there are duplicate values.
+         *       Which key wins is indeterminate:
+         * "{:a 1  :b 1} -> {1 :a} *or* {1 :b}"
+         */
+        private static <K, V> Map<V, K> simpleReverseMap(Map<K, V> map) {
+            Map<V, K> ret = new HashMap<V, K>();
+            for (Map.Entry<K, V> entry : map.entrySet()) {
+                ret.put(entry.getValue(), entry.getKey());
+            }
+            return ret;
+        }
+
         public IdDictionary(StormTopology topology) {
             List<String> componentNames = new ArrayList<>(topology.get_spouts().keySet());
             componentNames.addAll(topology.get_bolts().keySet());
@@ -143,7 +158,7 @@ public class SerializationFactory {
                 ComponentCommon common = Utils.getComponentCommon(topology, name);
                 List<String> streams = new ArrayList<>(common.get_streams().keySet());
                 streamNametoId.put(name, idify(streams));
-                streamIdToName.put(name, Utils.reverseMap(streamNametoId.get(name)));
+                streamIdToName.put(name, simpleReverseMap(streamNametoId.get(name)));
             }
         }
 

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java b/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
index 2f06102..0284725 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/FeederSpout.java
@@ -17,17 +17,18 @@
  */
 package org.apache.storm.testing;
 
-import org.apache.storm.topology.OutputFieldsDeclarer;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.UUID;
+
 import org.apache.storm.spout.SpoutOutputCollector;
 import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.OutputFieldsDeclarer;
 import org.apache.storm.topology.base.BaseRichSpout;
 import org.apache.storm.tuple.Fields;
 import org.apache.storm.tuple.Values;
 import org.apache.storm.utils.InprocMessaging;
-import java.util.HashMap;
-import java.util.List;
-import java.util.UUID;
 
 
 public class FeederSpout extends BaseRichSpout {
@@ -51,7 +52,15 @@ public class FeederSpout extends BaseRichSpout {
 
     public void feed(List<Object> tuple, Object msgId) {
         InprocMessaging.sendMessage(_id, new Values(tuple, msgId));
-    }    
+    }
+    
+    public void feedNoWait(List<Object> tuple, Object msgId) {
+        InprocMessaging.sendMessageNoWait(_id, new Values(tuple, msgId));
+    }
+    
+    public void waitForReader() {
+        InprocMessaging.waitForReader(_id);
+    }
     
     public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
         _collector = collector;
@@ -63,17 +72,11 @@ public class FeederSpout extends BaseRichSpout {
 
     public void nextTuple() {
         List<Object> toEmit = (List<Object>) InprocMessaging.pollMessage(_id);
-        if(toEmit!=null) {
+        if (toEmit!=null) {
             List<Object> tuple = (List<Object>) toEmit.get(0);
             Object msgId = toEmit.get(1);
             
             _collector.emit(tuple, msgId);
-        } else {
-            try {
-                Thread.sleep(1);
-            } catch (InterruptedException e) {
-                throw new RuntimeException(e);
-            }
         }
     }
 


[07/21] storm git commit: STORM-2018: Supervisor V2

Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/ZKStateStorage.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/ZKStateStorage.java b/storm-core/src/jvm/org/apache/storm/cluster/ZKStateStorage.java
new file mode 100644
index 0000000..e337b1f
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/ZKStateStorage.java
@@ -0,0 +1,252 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.state.*;
+import org.apache.curator.framework.state.ConnectionState;
+import org.apache.storm.Config;
+import org.apache.storm.callback.DefaultWatcherCallBack;
+import org.apache.storm.callback.WatcherCallBack;
+import org.apache.storm.callback.ZKStateChangedCallback;
+import org.apache.storm.utils.Utils;
+import org.apache.storm.zookeeper.Zookeeper;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.ACL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class ZKStateStorage implements IStateStorage {
+
+    private static Logger LOG = LoggerFactory.getLogger(ZKStateStorage.class);
+
+    private ConcurrentHashMap<String, ZKStateChangedCallback> callbacks = new ConcurrentHashMap<String, ZKStateChangedCallback>();
+    private CuratorFramework zkWriter;
+    private CuratorFramework zkReader;
+    private AtomicBoolean active;
+
+    private boolean isNimbus;
+    private Map authConf;
+    private Map<Object, Object> conf;
+
+    private class ZkWatcherCallBack implements WatcherCallBack{
+        @Override
+        public void execute(Watcher.Event.KeeperState state, Watcher.Event.EventType type, String path) {
+            if (active.get()) {
+                if (!(state.equals(Watcher.Event.KeeperState.SyncConnected))) {
+                    LOG.debug("Received event {} : {}: {} with disconnected Zookeeper.", state, type, path);
+                } else {
+                    LOG.debug("Received event {} : {} : {}", state, type, path);
+                }
+
+                if (!type.equals(Watcher.Event.EventType.None)) {
+                    for (Map.Entry<String, ZKStateChangedCallback> e : callbacks.entrySet()) {
+                        ZKStateChangedCallback fn = e.getValue();
+                        fn.changed(type, path);
+                    }
+                }
+            }
+        }
+    }
+
+    public ZKStateStorage(Map<Object, Object> conf, Map authConf, List<ACL> acls, ClusterStateContext context) throws Exception {
+        this.conf = conf;
+        this.authConf = authConf;
+        if (context.getDaemonType().equals(DaemonType.NIMBUS))
+            this.isNimbus = true;
+
+        // just mkdir STORM_ZOOKEEPER_ROOT dir
+        CuratorFramework zkTemp = mkZk();
+        String rootPath = String.valueOf(conf.get(Config.STORM_ZOOKEEPER_ROOT));
+        Zookeeper.mkdirs(zkTemp, rootPath, acls);
+        zkTemp.close();
+
+        active = new AtomicBoolean(true);
+        zkWriter = mkZk(new ZkWatcherCallBack());
+        if (isNimbus) {
+            zkReader = mkZk(new ZkWatcherCallBack());
+        } else {
+            zkReader = zkWriter;
+        }
+
+    }
+
+    @SuppressWarnings("unchecked")
+    private CuratorFramework mkZk() throws IOException {
+        return Zookeeper.mkClient(conf, (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS), conf.get(Config.STORM_ZOOKEEPER_PORT), "",
+                new DefaultWatcherCallBack(), authConf);
+    }
+
+    @SuppressWarnings("unchecked")
+    private CuratorFramework mkZk(WatcherCallBack watcher) throws NumberFormatException, IOException {
+        return Zookeeper.mkClient(conf, (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS), conf.get(Config.STORM_ZOOKEEPER_PORT),
+                String.valueOf(conf.get(Config.STORM_ZOOKEEPER_ROOT)), watcher, authConf);
+    }
+
+    @Override
+    public void delete_node_blobstore(String path, String nimbusHostPortInfo) {
+        Zookeeper.deleteNodeBlobstore(zkWriter, path, nimbusHostPortInfo);
+    }
+
+    @Override
+    public String register(ZKStateChangedCallback callback) {
+        String id = UUID.randomUUID().toString();
+        this.callbacks.put(id, callback);
+        return id;
+    }
+
+    @Override
+    public void unregister(String id) {
+        this.callbacks.remove(id);
+    }
+
+    @Override
+    public String create_sequential(String path, byte[] data, List<ACL> acls) {
+        return Zookeeper.createNode(zkWriter, path, data, CreateMode.EPHEMERAL_SEQUENTIAL, acls);
+    }
+
+    @Override
+    public void mkdirs(String path, List<ACL> acls) {
+        Zookeeper.mkdirs(zkWriter, path, acls);
+    }
+
+    @Override
+    public void delete_node(String path) {
+        Zookeeper.deleteNode(zkWriter, path);
+    }
+
+    @Override
+    public void set_ephemeral_node(String path, byte[] data, List<ACL> acls) {
+        Zookeeper.mkdirs(zkWriter, Zookeeper.parentPath(path), acls);
+        if (Zookeeper.exists(zkWriter, path, false)) {
+            try {
+                Zookeeper.setData(zkWriter, path, data);
+            } catch (RuntimeException e) {
+                if (Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) {
+                    Zookeeper.createNode(zkWriter, path, data, CreateMode.EPHEMERAL, acls);
+                } else {
+                    throw e;
+                }
+            }
+
+        } else {
+            Zookeeper.createNode(zkWriter, path, data, CreateMode.EPHEMERAL, acls);
+        }
+    }
+
+    @Override
+    public Integer get_version(String path, boolean watch) throws Exception {
+        Integer ret = Zookeeper.getVersion(zkReader, path, watch);
+        return ret;
+    }
+
+    @Override
+    public boolean node_exists(String path, boolean watch) {
+        return Zookeeper.existsNode(zkReader, path, watch);
+    }
+
+    @Override
+    public List<String> get_children(String path, boolean watch) {
+        return Zookeeper.getChildren(zkReader, path, watch);
+    }
+
+    @Override
+    public void close() {
+        this.active.set(false);
+        zkWriter.close();
+        if (isNimbus) {
+            zkReader.close();
+        }
+    }
+
+    @Override
+    public void set_data(String path, byte[] data, List<ACL> acls) {
+        if (Zookeeper.exists(zkWriter, path, false)) {
+            Zookeeper.setData(zkWriter, path, data);
+        } else {
+            Zookeeper.mkdirs(zkWriter, Zookeeper.parentPath(path), acls);
+            try {
+                Zookeeper.createNode(zkWriter, path, data, CreateMode.PERSISTENT, acls);
+            } catch (RuntimeException e) {
+                if (Utils.exceptionCauseIsInstanceOf(KeeperException.NodeExistsException.class, e)) {
+                    Zookeeper.setData(zkWriter, path, data);
+                } else {
+                    throw e;
+                }
+            }
+        }
+    }
+
+    @Override
+    public byte[] get_data(String path, boolean watch) {
+        byte[] ret = null;
+
+        ret = Zookeeper.getData(zkReader, path, watch);
+
+        return ret;
+    }
+
+    @Override
+    public VersionedData<byte[]> get_data_with_version(String path, boolean watch) {
+        return Zookeeper.getDataWithVersion(zkReader, path, watch);
+    }
+
+    @Override
+    public void set_worker_hb(String path, byte[] data, List<ACL> acls) {
+        set_data(path, data, acls);
+    }
+
+    @Override
+    public byte[] get_worker_hb(String path, boolean watch) {
+        return Zookeeper.getData(zkReader, path, watch);
+    }
+
+    @Override
+    public List<String> get_worker_hb_children(String path, boolean watch) {
+        return get_children(path, watch);
+    }
+
+    @Override
+    public void delete_worker_hb(String path) {
+        delete_node(path);
+    }
+
+    @Override
+    public void add_listener(final ConnectionStateListener listener) {
+        Zookeeper.addListener(zkReader, new ConnectionStateListener() {
+            @Override
+            public void stateChanged(CuratorFramework curatorFramework, ConnectionState connectionState) {
+                listener.stateChanged(curatorFramework, connectionState);
+            }
+        });
+    }
+
+    @Override
+    public void sync_path(String path) {
+        Zookeeper.syncPath(zkWriter, path);
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java b/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java
new file mode 100644
index 0000000..73bcea0
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/command/HealthCheck.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.command;
+
+import org.apache.storm.Config;
+import org.apache.storm.utils.Utils;
+
+import org.apache.storm.utils.ConfigUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class HealthCheck {
+
+    private static final Logger LOG = LoggerFactory.getLogger(HealthCheck.class);
+    private static final String FAILED = "failed";
+    private static final String SUCCESS = "success";
+    private static final String TIMEOUT = "timeout";
+    private static final String FAILED_WITH_EXIT_CODE = "failed_with_exit_code";
+
+    public static int healthCheck(Map conf) {
+        String healthDir = ConfigUtils.absoluteHealthCheckDir(conf);
+        List<String> results = new ArrayList<>();
+        if (healthDir != null) {
+            File parentFile = new File(healthDir);
+            List<String> healthScripts = new ArrayList<String>();
+            if (parentFile.exists()) {
+                File[] list = parentFile.listFiles();
+                for (File f : list) {
+                    if (!f.isDirectory() && f.canExecute())
+                        healthScripts.add(f.getAbsolutePath());
+                }
+            }
+            for (String script : healthScripts) {
+                String result = processScript(conf, script);
+                results.add(result);
+            }
+        }
+
+        // failed_with_exit_code is OK. We're mimicing Hadoop's health checks.
+        // We treat non-zero exit codes as indicators that the scripts failed
+        // to execute properly, not that the system is unhealthy, in which case
+        // we don't want to start killing things.
+
+        if (results.contains(FAILED) || results.contains(TIMEOUT)) {
+            return 1;
+        } else {
+            return 0;
+        }
+
+    }
+
+    public static String processScript(Map conf, String script) {
+        Thread interruptThread = null;
+        try {
+            Process process = Runtime.getRuntime().exec(script);
+            final long timeout = (long) (conf.get(Config.STORM_HEALTH_CHECK_TIMEOUT_MS));
+            final Thread curThread = Thread.currentThread();
+            // kill process when timeout
+            interruptThread = new Thread(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        Thread.sleep(timeout);
+                        curThread.interrupt();
+                    } catch (InterruptedException e) {
+                        // Ignored
+                    }
+                }
+            });
+            interruptThread.start();
+            process.waitFor();
+            interruptThread.interrupt();
+            curThread.interrupted();
+
+            if (process.exitValue() != 0) {
+                String str;
+                InputStream stdin = process.getInputStream();
+                BufferedReader reader = new BufferedReader(new InputStreamReader(stdin));
+                while ((str = reader.readLine()) != null) {
+                    if (str.startsWith("ERROR")) {
+                        return FAILED;
+                    }
+                }
+                return SUCCESS;
+            }
+            return FAILED_WITH_EXIT_CODE;
+        } catch (InterruptedException e) {
+            LOG.warn("Script:  {} timed out.", script);
+            return TIMEOUT;
+        } catch (Exception e) {
+            LOG.warn("Script failed with exception: ", e);
+            return FAILED_WITH_EXIT_CODE;
+        } finally {
+            if (interruptThread != null)
+                interruptThread.interrupt();
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        Map<String, Object> conf = ConfigUtils.readStormConfig();
+        System.exit(healthCheck(conf));
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java b/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java
new file mode 100644
index 0000000..1fcd81b
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/command/KillWorkers.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.command;
+
+import java.io.File;
+import java.util.Map;
+
+import org.apache.storm.Config;
+import org.apache.storm.utils.Utils;
+
+import org.apache.storm.daemon.supervisor.StandaloneSupervisor;
+import org.apache.storm.daemon.supervisor.Supervisor;
+import org.apache.storm.utils.ConfigUtils;
+
+public class KillWorkers {
+    public static void main(String [] args) throws Exception {
+        Map<String, Object> conf = ConfigUtils.readStormConfig();
+        conf.put(Config.STORM_LOCAL_DIR, new File((String)conf.get(Config.STORM_LOCAL_DIR)).getCanonicalPath());
+        try (Supervisor supervisor = new Supervisor(conf, null, new StandaloneSupervisor())) {
+            supervisor.shutdownAllWorkers();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/DaemonCommon.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/DaemonCommon.java b/storm-core/src/jvm/org/apache/storm/daemon/DaemonCommon.java
new file mode 100644
index 0000000..d1b71a7
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/DaemonCommon.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon;
+
+public interface DaemonCommon {
+    public boolean isWaiting();
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/metrics/MetricsUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/metrics/MetricsUtils.java b/storm-core/src/jvm/org/apache/storm/daemon/metrics/MetricsUtils.java
index 56b920b..eb72939 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/metrics/MetricsUtils.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/metrics/MetricsUtils.java
@@ -20,6 +20,7 @@ package org.apache.storm.daemon.metrics;
 import org.apache.storm.Config;
 import org.apache.storm.daemon.metrics.reporters.JmxPreparableReporter;
 import org.apache.storm.daemon.metrics.reporters.PreparableReporter;
+import org.apache.storm.utils.ConfigUtils;
 import org.apache.storm.utils.Utils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -85,8 +86,8 @@ public class MetricsUtils {
     public static File getCsvLogDir(Map stormConf) {
         String csvMetricsLogDirectory = Utils.getString(stormConf.get(Config.STORM_DAEMON_METRICS_REPORTER_CSV_LOG_DIR), null);
         if (csvMetricsLogDirectory == null) {
-            csvMetricsLogDirectory = absoluteStormLocalDir(stormConf);
-            csvMetricsLogDirectory = csvMetricsLogDirectory + File.separator + "csvmetrics";
+            csvMetricsLogDirectory = ConfigUtils.absoluteStormLocalDir(stormConf);
+            csvMetricsLogDirectory = csvMetricsLogDirectory + ConfigUtils.FILE_SEPARATOR + "csvmetrics";
         }
         File csvMetricsDir = new File(csvMetricsLogDirectory);
         validateCreateOutputDir(csvMetricsDir);
@@ -104,18 +105,4 @@ public class MetricsUtils {
             throw new IllegalStateException(dir.getName() + " is not a directory.");
         }
     }
-
-    public static String absoluteStormLocalDir(Map conf) {
-        String stormHome = System.getProperty("storm.home");
-        String localDir = (String) conf.get(Config.STORM_LOCAL_DIR);
-        if (localDir == null) {
-            return (stormHome + File.separator + "storm-local");
-        } else {
-            if (new File(localDir).isAbsolute()) {
-                return localDir;
-            } else {
-                return (stormHome + File.separator + localDir);
-            }
-        }
-    }
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
new file mode 100644
index 0000000..361328e
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/AdvancedFSOps.java
@@ -0,0 +1,335 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.storm.Config;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AdvancedFSOps {
+    private static final Logger LOG = LoggerFactory.getLogger(AdvancedFSOps.class);
+    
+    /**
+     * Factory to create a new AdvancedFSOps
+     * @param conf the configuration of the process
+     * @return the appropriate instance of the class for this config and environment.
+     */
+    public static AdvancedFSOps make(Map<String, Object> conf) {
+        if (Utils.isOnWindows()) {
+            return new AdvancedWindowsFSOps(conf);
+        }
+        if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
+            return new AdvancedRunAsUserFSOps(conf);
+        }
+        return new AdvancedFSOps();
+    }
+    
+    private static class AdvancedRunAsUserFSOps extends AdvancedFSOps {
+        private final Map<String, Object> _conf;
+        
+        public AdvancedRunAsUserFSOps(Map<String, Object> conf) {
+            if (Utils.isOnWindows()) {
+                throw new UnsupportedOperationException("ERROR: Windows doesn't support running workers as different users yet");
+            }
+            _conf = conf;
+        }
+        
+        @Override
+        public void setupBlobPermissions(File path, String user) throws IOException {
+            String logPrefix = "setup blob permissions for " + path;
+            SupervisorUtils.processLauncherAndWait(_conf, user, Arrays.asList("blob", path.toString()), null, logPrefix);
+        }
+        
+        @Override
+        public void deleteIfExists(File path, String user, String logPrefix) throws IOException {
+            String absolutePath = path.getAbsolutePath();
+            LOG.info("Deleting path {}", absolutePath);
+            if (user == null) {
+                user = Files.getOwner(path.toPath()).getName();
+            }
+            List<String> commands = new ArrayList<>();
+            commands.add("rmr");
+            commands.add(absolutePath);
+            SupervisorUtils.processLauncherAndWait(_conf, user, commands, null, logPrefix);
+            if (Utils.checkFileExists(absolutePath)) {
+                throw new RuntimeException(path + " was not deleted.");
+            }
+        }
+        
+        @Override
+        public void setupStormCodeDir(Map<String, Object> topologyConf, File path) throws IOException {
+            SupervisorUtils.setupStormCodeDir(_conf, topologyConf, path.getCanonicalPath());
+        }
+    }
+    
+    /**
+     * Operations that need to override the default ones when running on Windows
+     *
+     */
+    private static class AdvancedWindowsFSOps extends AdvancedFSOps {
+
+        public AdvancedWindowsFSOps(Map<String, Object> conf) {
+            if (Utils.getBoolean(conf.get(Config.SUPERVISOR_RUN_WORKER_AS_USER), false)) {
+                throw new RuntimeException("ERROR: Windows doesn't support running workers as different users yet");
+            }
+        }
+        
+        @Override
+        public void restrictDirectoryPermissions(File dir) throws IOException {
+            //NOOP, if windows gets support for run as user we will need to find a way to support this
+        }
+        
+        @Override
+        public void moveDirectoryPreferAtomic(File fromDir, File toDir) throws IOException {
+            // Files/move with non-empty directory doesn't work well on Windows
+            // This is not atomic but it does work
+            FileUtils.moveDirectory(fromDir, toDir);
+        }
+        
+        @Override
+        public boolean supportsAtomicDirectoryMove() {
+            // Files/move with non-empty directory doesn't work well on Windows
+            // FileUtils.moveDirectory is not atomic
+            return false;
+        }
+    }
+    
+    
+    protected AdvancedFSOps() {
+        //NOOP, but restricted permissions
+    }
+
+    /**
+     * Set directory permissions to (OWNER)RWX (GROUP)R-X (OTHER)---
+     * On some systems that do not support this, it may become a noop
+     * @param dir the directory to change permissions on
+     * @throws IOException on any error
+     */
+    public void restrictDirectoryPermissions(File dir) throws IOException {
+        Set<PosixFilePermission> perms = new HashSet<>(
+                Arrays.asList(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE,
+                        PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_READ,
+                        PosixFilePermission.GROUP_EXECUTE));
+        Files.setPosixFilePermissions(dir.toPath(), perms);
+    }
+
+    /**
+     * Move fromDir to toDir, and try to make it an atomic move if possible
+     * @param fromDir what to move
+     * @param toDir where to move it from
+     * @throws IOException on any error
+     */
+    public void moveDirectoryPreferAtomic(File fromDir, File toDir) throws IOException {
+        FileUtils.forceMkdir(toDir);
+        Files.move(fromDir.toPath(), toDir.toPath(), StandardCopyOption.ATOMIC_MOVE);
+    }
+    
+    /**
+     * @return true if an atomic directory move works, else false.
+     */
+    public boolean supportsAtomicDirectoryMove() {
+        return true;
+    }
+    
+    /**
+     * Copy a directory
+     * @param fromDir from where
+     * @param toDir to where
+     * @throws IOException on any error
+     */
+    public void copyDirectory(File fromDir, File toDir) throws IOException {
+        FileUtils.copyDirectory(fromDir, toDir);
+    }
+    
+    /**
+     * Setup permissions properly for an internal blob store path
+     * @param path the path to set the permissions on
+     * @param user the user to change the permissions for
+     * @throws IOException on any error
+     */
+    public void setupBlobPermissions(File path, String user) throws IOException {
+        //Normally this is a NOOP
+    }
+
+    /**
+     * Delete a file or a directory and all of the children. If it exists.
+     * @param path what to delete
+     * @param user who to delete it as if doing it as someone else is supported
+     * @param logPrefix if an external process needs to be launched to delete 
+     * the object what prefix to include in the logs
+     * @throws IOException on any error.
+     */
+    public void deleteIfExists(File path, String user, String logPrefix) throws IOException {
+        //by default no need to do this as a different user
+        deleteIfExists(path);
+    }
+    
+    /**
+     * Delete a file or a directory and all of the children. If it exists.
+     * @param path what to delete
+     * @throws IOException on any error.
+     */
+    public void deleteIfExists(File path) throws IOException {
+        LOG.info("Deleting path {}", path);
+        Path p = path.toPath();
+        if (Files.exists(p)) {
+            try {
+                FileUtils.forceDelete(path);
+            } catch (FileNotFoundException ignored) {}
+        }
+    }
+
+    /**
+     * Setup the permissions for the storm code dir
+     * @param topologyConf the config of the Topology
+     * @param path the directory to set the permissions on
+     * @throws IOException on any error
+     */
+    public void setupStormCodeDir(Map<String, Object> topologyConf, File path) throws IOException {
+        //By default this is a NOOP
+    }
+
+    /**
+     * Sanity check if everything the topology needs is there for it to run.
+     * @param conf the config of the supervisor
+     * @param topologyId the ID of the topology
+     * @return true if everything is there, else false
+     * @throws IOException on any error
+     */
+    public boolean doRequiredTopoFilesExist(Map<String, Object> conf, String topologyId) throws IOException {
+        return SupervisorUtils.doRequiredTopoFilesExist(conf, topologyId);
+    }
+    
+    /**
+     * Makes a directory, including any necessary but nonexistent parent
+     * directories. 
+     *
+     * @param path the directory to create
+     * @throws IOException on any error
+     */
+    public void forceMkdir(File path) throws IOException {
+        FileUtils.forceMkdir(path);
+    }
+    
+    /**
+     * Check if a file exists or not
+     * @param path the path to check
+     * @return true if it exists else false
+     * @throws IOException on any error.
+     */
+    public boolean fileExists(File path) throws IOException {
+        return path.exists();
+    }
+
+    /**
+     * Get a writer for the given location 
+     * @param file the file to write to
+     * @return the Writer to use.
+     * @throws IOException on any error
+     */
+    public Writer getWriter(File file) throws IOException {
+        return new FileWriter(file);
+    }
+    
+    /**
+     * Get an output stream to write to a given file
+     * @param file the file to write to
+     * @return an OutputStream for that file
+     * @throws IOException on any error
+     */
+    public OutputStream getOutputStream(File file) throws IOException {
+        return new FileOutputStream(file);
+    }
+    
+    /**
+     * Dump a string to a file
+     * @param location where to write to
+     * @param data the data to write
+     * @throws IOException on any error
+     */
+    public void dump(File location, String data) throws IOException {
+        File parent = location.getParentFile();
+        if (!parent.exists()) {
+            forceMkdir(parent);
+        }
+        try (Writer w = getWriter(location)) {
+            w.write(data);
+        }
+    }
+    
+    /**
+     * Read the contents of a file into a String
+     * @param location the file to read
+     * @return the contents of the file
+     * @throws IOException on any error
+     */
+    public String slurpString(File location) throws IOException {
+        return FileUtils.readFileToString(location, "UTF-8");
+    }
+
+    /**
+     * Read the contents of a file into a byte array.
+     * @param localtion the file to read
+     * @return the contents of the file
+     * @throws IOException on any error
+     */ 
+    public byte[] slurp(File location) throws IOException {
+        return FileUtils.readFileToByteArray(location);
+    }
+
+    /**
+     * Create a symbolic link pointing at target
+     * @param link the link to create
+     * @param target where it should point to
+     * @throws IOException on any error.
+     */
+    public void createSymlink(File link, File target) throws IOException {
+        Path plink = link.toPath().toAbsolutePath();
+        Path ptarget = target.toPath().toAbsolutePath();
+        LOG.debug("Creating symlink [{}] to [{}]", plink, ptarget);
+        if (Files.exists(plink)) {
+            if (Files.isSameFile(plink, ptarget)) {
+                //It already points where we want it to
+                return;
+            }
+            FileUtils.forceDelete(link);
+        }
+        Files.createSymbolicLink(plink, ptarget);
+    }
+    
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
new file mode 100644
index 0000000..237d89b
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -0,0 +1,638 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.storm.Config;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.generated.WorkerResources;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+
+/**
+ * A container that runs processes on the local box.
+ */
+public class BasicContainer extends Container {
+    private static final Logger LOG = LoggerFactory.getLogger(BasicContainer.class);
+    private static final FilenameFilter jarFilter = (dir, name) -> name.endsWith(".jar");
+    private static final Joiner CPJ = 
+            Joiner.on(Utils.CLASS_PATH_SEPARATOR).skipNulls();
+    
+    protected final LocalState _localState;
+    protected final String _profileCmd;
+    protected final String _stormHome = System.getProperty("storm.home");
+    protected volatile boolean _exitedEarly = false;
+
+    private class ProcessExitCallback implements ExitCodeCallback {
+        private final String _logPrefix;
+
+        public ProcessExitCallback(String logPrefix) {
+            _logPrefix = logPrefix;
+        }
+
+        @Override
+        public void call(int exitCode) {
+            LOG.info("{} exited with code: {}", _logPrefix, exitCode);
+            _exitedEarly = true;
+        }
+    }
+    
+    /**
+     * Create a new BasicContainer
+     * @param type the type of container being made.
+     * @param conf the supervisor config
+     * @param supervisorId the ID of the supervisor this is a part of.
+     * @param port the port the container is on.  Should be <= 0 if only a partial recovery
+     * @param assignment the assignment for this container. Should be null if only a partial recovery.
+     * @param localState the local state of the supervisor.  May be null if partial recovery
+     * @param workerId the id of the worker to use.  Must not be null if doing a partial recovery.
+     */
+    public BasicContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+            LocalAssignment assignment,
+            LocalState localState, String workerId) throws IOException {
+        this(type, conf, supervisorId, port, assignment, localState, workerId, null, null, null);
+    }
+    
+    /**
+     * Create a new BasicContainer
+     * @param type the type of container being made.
+     * @param conf the supervisor config
+     * @param supervisorId the ID of the supervisor this is a part of.
+     * @param port the port the container is on.  Should be <= 0 if only a partial recovery
+     * @param assignment the assignment for this container. Should be null if only a partial recovery.
+     * @param localState the local state of the supervisor.  May be null if partial recovery
+     * @param workerId the id of the worker to use.  Must not be null if doing a partial recovery.
+     * @param ops file system operations (mostly for testing) if null a new one is made
+     * @param topoConf the config of the topology (mostly for testing) if null 
+     * and not a partial recovery the real conf is read.
+     * @param profileCmd the command to use when profiling (used for testing)
+     * @throws IOException on any error
+     * @throws ContainerRecoveryException if the Container could not be recovered.
+     */
+    BasicContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+            LocalAssignment assignment,
+            LocalState localState, String workerId, Map<String, Object> topoConf, 
+            AdvancedFSOps ops, String profileCmd) throws IOException {
+        super(type, conf, supervisorId, port, assignment, workerId, topoConf, ops);
+        assert(localState != null);
+        _localState = localState;
+
+        if (type.isRecovery() && !type.isOnlyKillable()) {
+            synchronized (localState) {
+                String wid = null;
+                Map<String, Integer> workerToPort = localState.getApprovedWorkers();
+                for (Map.Entry<String, Integer> entry : workerToPort.entrySet()) {
+                    if (port == entry.getValue().intValue()) {
+                        wid = entry.getKey();
+                    }
+                }
+                if (wid == null) {
+                    throw new ContainerRecoveryException("Could not find worker id for " + port + " " + assignment);
+                }
+                LOG.info("Recovered Worker {}", wid);
+                _workerId = wid;
+            }
+        } else if (_workerId == null){
+            createNewWorkerId();
+        }
+
+        if (profileCmd == null) {
+            profileCmd = _stormHome + Utils.FILE_PATH_SEPARATOR + "bin" + Utils.FILE_PATH_SEPARATOR
+                    + conf.get(Config.WORKER_PROFILER_COMMAND);
+        }
+        _profileCmd = profileCmd;
+    }
+
+    /**
+     * Create a new worker ID for this process and store in in this object and
+     * in the local state.  Never call this if a worker is currently up and running.
+     * We will lose track of the process.
+     */
+    protected void createNewWorkerId() {
+        _type.assertFull();
+        assert(_workerId == null);
+        synchronized (_localState) {
+            _workerId = Utils.uuid();
+            Map<String, Integer> workerToPort = _localState.getApprovedWorkers();
+            if (workerToPort == null) {
+                workerToPort = new HashMap<>(1);
+            }
+            removeWorkersOn(workerToPort, _port);
+            workerToPort.put(_workerId, _port);
+            _localState.setApprovedWorkers(workerToPort);
+            LOG.info("Created Worker ID {}", _workerId);
+        }
+    }
+
+    private static void removeWorkersOn(Map<String, Integer> workerToPort, int _port) {
+        for (Iterator<Entry<String, Integer>> i = workerToPort.entrySet().iterator(); i.hasNext();) {
+            Entry<String, Integer> found = i.next();
+            if (_port == found.getValue().intValue()) {
+                LOG.warn("Deleting worker {} from state", found.getKey());
+                i.remove();
+            }
+        }
+    }
+
+    @Override
+    public void cleanUpForRestart() throws IOException {
+        String origWorkerId = _workerId;
+        super.cleanUpForRestart();
+        synchronized (_localState) {
+            Map<String, Integer> workersToPort = _localState.getApprovedWorkers();
+            workersToPort.remove(origWorkerId);
+            removeWorkersOn(workersToPort, _port);
+            _localState.setApprovedWorkers(workersToPort);
+            LOG.info("Removed Worker ID {}", origWorkerId);
+        }
+    }
+
+    @Override
+    public void relaunch() throws IOException {
+        _type.assertFull();
+        //We are launching it now...
+        _type = ContainerType.LAUNCH;
+        createNewWorkerId();
+        setup();
+        launch();
+    }
+
+    @Override
+    public boolean didMainProcessExit() {
+        return _exitedEarly;
+    }
+
+    /**
+     * Run the given command for profiling
+     * 
+     * @param command
+     *            the command to run
+     * @param env
+     *            the environment to run the command
+     * @param logPrefix
+     *            the prefix to include in the logs
+     * @param targetDir
+     *            the working directory to run the command in
+     * @return true if it ran successfully, else false
+     * @throws IOException
+     *             on any error
+     * @throws InterruptedException
+     *             if interrupted wile waiting for the process to exit.
+     */
+    protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix,
+            File targetDir) throws IOException, InterruptedException {
+        _type.assertFull();
+        Process p = SupervisorUtils.launchProcess(command, env, logPrefix, null, targetDir);
+        int ret = p.waitFor();
+        return ret == 0;
+    }
+
+    @Override
+    public boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException {
+        _type.assertFull();
+        String targetDir = ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port);
+
+        @SuppressWarnings("unchecked")
+        Map<String, String> env = (Map<String, String>) _topoConf.get(Config.TOPOLOGY_ENVIRONMENT);
+        if (env == null) {
+            env = new HashMap<String, String>();
+        }
+
+        String str = ConfigUtils.workerArtifactsPidPath(_conf, _topologyId, _port);
+
+        String workerPid = _ops.slurpString(new File(str)).trim();
+
+        ProfileAction profileAction = request.get_action();
+        String logPrefix = "ProfilerAction process " + _topologyId + ":" + _port + " PROFILER_ACTION: " + profileAction
+                + " ";
+
+        List<String> command = mkProfileCommand(profileAction, stop, workerPid, targetDir);
+
+        File targetFile = new File(targetDir);
+        if (command.size() > 0) {
+            return runProfilingCommand(command, env, logPrefix, targetFile);
+        }
+        LOG.warn("PROFILING REQUEST NOT SUPPORTED {} IGNORED...", request);
+        return true;
+    }
+
+    /**
+     * Get the command to run when doing profiling
+     * @param action the profiling action to perform
+     * @param stop if this is meant to stop the profiling or start it
+     * @param workerPid the PID of the process to profile
+     * @param targetDir the current working directory of the worker process
+     * @return the command to run for profiling.
+     */
+    private List<String> mkProfileCommand(ProfileAction action, boolean stop, String workerPid, String targetDir) {
+        switch(action) {
+            case JMAP_DUMP:
+                return jmapDumpCmd(workerPid, targetDir);
+            case JSTACK_DUMP:
+                return jstackDumpCmd(workerPid, targetDir);
+            case JPROFILE_DUMP:
+                return jprofileDump(workerPid, targetDir);
+            case JVM_RESTART:
+                return jprofileJvmRestart(workerPid);
+            case JPROFILE_STOP:
+                if (stop) {
+                    return jprofileStop(workerPid, targetDir);
+                }
+                return jprofileStart(workerPid);
+            default:
+                return Lists.newArrayList();
+        }
+    }
+
+    private List<String> jmapDumpCmd(String pid, String targetDir) {
+        return Lists.newArrayList(_profileCmd, pid, "jmap", targetDir);
+    }
+
+    private List<String> jstackDumpCmd(String pid, String targetDir) {
+        return Lists.newArrayList(_profileCmd, pid, "jstack", targetDir);
+    }
+
+    private List<String> jprofileStart(String pid) {
+        return Lists.newArrayList(_profileCmd, pid, "start");
+    }
+
+    private List<String> jprofileStop(String pid, String targetDir) {
+        return Lists.newArrayList(_profileCmd, pid, "stop", targetDir);
+    }
+
+    private List<String> jprofileDump(String pid, String targetDir) {
+        return Lists.newArrayList(_profileCmd, pid, "dump", targetDir);
+    }
+
+    private List<String> jprofileJvmRestart(String pid) {
+        return Lists.newArrayList(_profileCmd, pid, "kill");
+    }
+
+    /**
+     * Compute the java.library.path that should be used for the worker.
+     * This helps it to load JNI libraries that are packaged in the uber jar.
+     * @param stormRoot the root directory of the worker process
+     * @param conf the config for the supervisor.
+     * @return the java.library.path/LD_LIBRARY_PATH to use so native libraries load correctly.
+     */
+    protected String javaLibraryPath(String stormRoot, Map<String, Object> conf) {
+        String resourceRoot = stormRoot + Utils.FILE_PATH_SEPARATOR + ConfigUtils.RESOURCES_SUBDIR;
+        String os = System.getProperty("os.name").replaceAll("\\s+", "_");
+        String arch = System.getProperty("os.arch");
+        String archResourceRoot = resourceRoot + Utils.FILE_PATH_SEPARATOR + os + "-" + arch;
+        String ret = CPJ.join(archResourceRoot, resourceRoot,
+                conf.get(Config.JAVA_LIBRARY_PATH));
+        return ret;
+    }
+
+    /**
+     * Returns a collection of jar file names found under the given directory.
+     * @param dir the directory to search
+     * @return the jar file names
+     */
+    protected List<String> getFullJars(File dir) {
+        File[] files = dir.listFiles(jarFilter);
+
+        if (files == null) {
+            return Collections.emptyList();
+        }
+
+        return Arrays.stream(files).map(f -> f.getAbsolutePath())
+                .collect(Collectors.toList());
+    }
+    
+    protected List<String> frameworkClasspath() {
+        File stormLibDir = new File(_stormHome, "lib");
+        String stormConfDir =
+                System.getenv("STORM_CONF_DIR") != null ?
+                System.getenv("STORM_CONF_DIR") :
+                new File(_stormHome, "conf").getAbsolutePath();
+        File stormExtlibDir = new File(_stormHome, "extlib");
+        String extcp = System.getenv("STORM_EXT_CLASSPATH");
+        List<String> pathElements = new LinkedList<>();
+        pathElements.addAll(getFullJars(stormLibDir));
+        pathElements.addAll(getFullJars(stormExtlibDir));
+        pathElements.add(extcp);
+        pathElements.add(stormConfDir);
+
+        return pathElements;
+    }
+    
+    @SuppressWarnings("unchecked")
+    private List<String> asStringList(Object o) {
+        if (o instanceof String) {
+            return Arrays.asList((String)o);
+        } else if (o instanceof List) {
+            return (List<String>)o;
+        }
+        return Collections.EMPTY_LIST;
+    }
+    
+    /**
+     * Compute the classpath for the worker process
+     * @param stormJar the topology jar
+     * @param dependencyLocations any dependencies from the topology
+     * @return the full classpath
+     */
+    protected String getWorkerClassPath(String stormJar, List<String> dependencyLocations) {
+        List<String> workercp = new ArrayList<>();
+        workercp.addAll(frameworkClasspath());
+        workercp.add(stormJar);
+        workercp.addAll(dependencyLocations);
+        workercp.addAll(asStringList(_topoConf.get(Config.TOPOLOGY_CLASSPATH)));
+        return CPJ.join(workercp);
+    }
+
+    private String substituteChildOptsInternal(String string, int memOnheap) {
+        if (StringUtils.isNotBlank(string)) {
+            String p = String.valueOf(_port);
+            string = string.replace("%ID%", p);
+            string = string.replace("%WORKER-ID%", _workerId);
+            string = string.replace("%TOPOLOGY-ID%", _topologyId);
+            string = string.replace("%WORKER-PORT%", p);
+            if (memOnheap > 0) {
+                string = string.replace("%HEAP-MEM%", String.valueOf(memOnheap));
+            }
+        }
+        return string;
+    }
+    
+    protected List<String> substituteChildopts(Object value) {
+        return substituteChildopts(value, -1);
+    }
+
+    protected List<String> substituteChildopts(Object value, int memOnheap) {
+        List<String> rets = new ArrayList<>();
+        if (value instanceof String) {
+            String string = substituteChildOptsInternal((String) value, memOnheap);
+            if (StringUtils.isNotBlank(string)) {
+                String[] strings = string.split("\\s+");
+                rets.addAll(Arrays.asList(strings));
+            }
+        } else if (value instanceof List) {
+            @SuppressWarnings("unchecked")
+            List<String> objects = (List<String>) value;
+            for (String object : objects) {
+                String str = substituteChildOptsInternal(object, memOnheap);
+                if (StringUtils.isNotBlank(str)) {
+                    rets.add(str);
+                }
+            }
+        }
+        return rets;
+    }
+
+    /**
+     * Launch the worker process (non-blocking)
+     * 
+     * @param command
+     *            the command to run
+     * @param env
+     *            the environment to run the command
+     * @param processExitcallback
+     *            a callback for when the process exits
+     * @param logPrefix
+     *            the prefix to include in the logs
+     * @param targetDir
+     *            the working directory to run the command in
+     * @return true if it ran successfully, else false
+     * @throws IOException
+     *             on any error
+     */
+    protected void launchWorkerProcess(List<String> command, Map<String, String> env, String logPrefix,
+            ExitCodeCallback processExitCallback, File targetDir) throws IOException {
+        SupervisorUtils.launchProcess(command, env, logPrefix, processExitCallback, targetDir);
+    }
+
+    private String getWorkerLoggingConfigFile() {
+        String log4jConfigurationDir = (String) (_conf.get(Config.STORM_LOG4J2_CONF_DIR));
+
+        if (StringUtils.isNotBlank(log4jConfigurationDir)) {
+            if (!Utils.isAbsolutePath(log4jConfigurationDir)) {
+                log4jConfigurationDir = _stormHome + Utils.FILE_PATH_SEPARATOR + log4jConfigurationDir;
+            }
+        } else {
+            log4jConfigurationDir = _stormHome + Utils.FILE_PATH_SEPARATOR + "log4j2";
+        }
+ 
+        if (Utils.IS_ON_WINDOWS && !log4jConfigurationDir.startsWith("file:")) {
+            log4jConfigurationDir = "file:///" + log4jConfigurationDir;
+        }
+        return log4jConfigurationDir + Utils.FILE_PATH_SEPARATOR + "worker.xml";
+    }
+    
+    /**
+     * Get parameters for the class path of the worker process.  Also used by the
+     * log Writer
+     * @param stormRoot the root dist dir for the topology
+     * @return the classpath for the topology as command line arguments.
+     * @throws IOException on any error.
+     */
+    private List<String> getClassPathParams(final String stormRoot) throws IOException {
+        final String stormJar = ConfigUtils.supervisorStormJarPath(stormRoot);
+        final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _ops);
+        final List<String> dependencyLocations = new ArrayList<>();
+        if (stormTopology.get_dependency_jars() != null) {
+            for (String dependency : stormTopology.get_dependency_jars()) {
+                dependencyLocations.add(new File(stormRoot, dependency).getAbsolutePath());
+            }
+        }
+
+        if (stormTopology.get_dependency_artifacts() != null) {
+            for (String dependency : stormTopology.get_dependency_artifacts()) {
+                dependencyLocations.add(new File(stormRoot, dependency).getAbsolutePath());
+            }
+        }
+        final String workerClassPath = getWorkerClassPath(stormJar, dependencyLocations);
+        
+        List<String> classPathParams = new ArrayList<>();
+        classPathParams.add("-cp");
+        classPathParams.add(workerClassPath);
+        return classPathParams;
+    }
+    
+    /**
+     * Get a set of java properties that are common to both the log writer and the worker processes.
+     * These are mostly system properties that are used by logging.
+     * @return a list of command line options
+     */
+    private List<String> getCommonParams() {
+        final String workersArtifacts = ConfigUtils.workerArtifactsRoot(_conf);
+        String stormLogDir = ConfigUtils.getLogDir();
+        String log4jConfigurationFile = getWorkerLoggingConfigFile();
+        
+        List<String> commonParams = new ArrayList<>();
+        commonParams.add("-Dlogging.sensitivity=" + OR((String) _topoConf.get(Config.TOPOLOGY_LOGGING_SENSITIVITY), "S3"));
+        commonParams.add("-Dlogfile.name=worker.log");
+        commonParams.add("-Dstorm.home=" + OR(_stormHome, ""));
+        commonParams.add("-Dworkers.artifacts=" + workersArtifacts);
+        commonParams.add("-Dstorm.id=" + _topologyId);
+        commonParams.add("-Dworker.id=" + _workerId);
+        commonParams.add("-Dworker.port=" + _port);
+        commonParams.add("-Dstorm.log.dir=" + stormLogDir);
+        commonParams.add("-Dlog4j.configurationFile=" + log4jConfigurationFile);
+        commonParams.add("-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector");
+        commonParams.add("-Dstorm.local.dir=" + _conf.get(Config.STORM_LOCAL_DIR));
+        return commonParams;
+    }
+    
+    private int getMemOnHeap(WorkerResources resources) {
+        int memOnheap = 0;
+        if (resources != null && resources.is_set_mem_on_heap() && 
+                resources.get_mem_on_heap() > 0) {
+            memOnheap = (int) Math.ceil(resources.get_mem_on_heap());
+        } else {
+            // set the default heap memory size for supervisor-test
+            memOnheap = Utils.getInt(_topoConf.get(Config.WORKER_HEAP_MEMORY_MB), 768);
+        }
+        return memOnheap;
+    }
+    
+    private List<String> getWorkerProfilerChildOpts(int memOnheap) {
+        List<String> workerProfilerChildopts = new ArrayList<>();
+        if (Utils.getBoolean(_conf.get(Config.WORKER_PROFILER_ENABLED), false)) {
+            workerProfilerChildopts = substituteChildopts(_conf.get(Config.WORKER_PROFILER_CHILDOPTS), memOnheap);
+        }
+        return workerProfilerChildopts;
+    }
+    
+    /**
+     * a or b the first one that is not null
+     * @param a something
+     * @param b something else
+     * @return a or b the first one that is not null
+     */
+    private <V> V OR(V a, V b) {
+        return a == null ? b : a;
+    }
+    
+    protected String javaCmd(String cmd) {
+        String ret = null;
+        String javaHome = System.getenv().get("JAVA_HOME");
+        if (StringUtils.isNotBlank(javaHome)) {
+            ret = javaHome + Utils.FILE_PATH_SEPARATOR + "bin" + Utils.FILE_PATH_SEPARATOR + cmd;
+        } else {
+            ret = cmd;
+        }
+        return ret;
+    }
+    
+    /**
+     * Create the command to launch the worker process
+     * @param memOnheap the on heap memory for the worker
+     * @param stormRoot the root dist dir for the topology
+     * @param jlp java library path for the topology
+     * @return the command to run
+     * @throws IOException on any error.
+     */
+    private List<String> mkLaunchCommand(final int memOnheap, final String stormRoot,
+            final String jlp) throws IOException {
+        final String javaCmd = javaCmd("java");
+        final String stormOptions = ConfigUtils.concatIfNotNull(System.getProperty("storm.options"));
+        final String stormConfFile = ConfigUtils.concatIfNotNull(System.getProperty("storm.conf.file"));
+        final String workerTmpDir = ConfigUtils.workerTmpRoot(_conf, _workerId);
+        
+        List<String> classPathParams = getClassPathParams(stormRoot);
+        List<String> commonParams = getCommonParams();
+        
+        List<String> commandList = new ArrayList<>();
+        //Log Writer Command...
+        commandList.add(javaCmd);
+        commandList.addAll(classPathParams);
+        commandList.addAll(substituteChildopts(_topoConf.get(Config.TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS)));
+        commandList.addAll(commonParams);
+        commandList.add("org.apache.storm.LogWriter"); //The LogWriter in turn launches the actual worker.
+
+        //Worker Command...
+        commandList.add(javaCmd);
+        commandList.add("-server");
+        commandList.addAll(commonParams);
+        commandList.addAll(substituteChildopts(_conf.get(Config.WORKER_CHILDOPTS), memOnheap));
+        commandList.addAll(substituteChildopts(_topoConf.get(Config.TOPOLOGY_WORKER_CHILDOPTS), memOnheap));
+        commandList.addAll(substituteChildopts(OR(
+                _topoConf.get(Config.TOPOLOGY_WORKER_GC_CHILDOPTS),
+                _conf.get(Config.WORKER_GC_CHILDOPTS)), memOnheap));
+        commandList.addAll(getWorkerProfilerChildOpts(memOnheap));
+        commandList.add("-Djava.library.path=" + jlp);
+        commandList.add("-Dstorm.conf.file=" + stormConfFile);
+        commandList.add("-Dstorm.options=" + stormOptions);
+        commandList.add("-Djava.io.tmpdir=" + workerTmpDir);
+        commandList.addAll(classPathParams);
+        commandList.add("org.apache.storm.daemon.worker");
+        commandList.add(_topologyId);
+        commandList.add(_supervisorId);
+        commandList.add(String.valueOf(_port));
+        commandList.add(_workerId);
+        
+        return commandList;
+    }
+
+    @Override
+    public void launch() throws IOException {
+        _type.assertFull();
+        LOG.info("Launching worker with assignment {} for this supervisor {} on port {} with id {}", _assignment,
+                _supervisorId, _port, _workerId);
+        String logPrefix = "Worker Process " + _workerId;
+        ProcessExitCallback processExitCallback = new ProcessExitCallback(logPrefix);
+        _exitedEarly = false;
+        
+        final WorkerResources resources = _assignment.get_resources();
+        final int memOnheap = getMemOnHeap(resources);
+        final String stormRoot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId);
+        final String jlp = javaLibraryPath(stormRoot, _conf);
+        
+        List<String> commandList = mkLaunchCommand(memOnheap, stormRoot, jlp);
+
+        Map<String, String> topEnvironment = new HashMap<String, String>();
+        @SuppressWarnings("unchecked")
+        Map<String, String> environment = (Map<String, String>) _topoConf.get(Config.TOPOLOGY_ENVIRONMENT);
+        if (environment != null) {
+            topEnvironment.putAll(environment);
+        }
+        topEnvironment.put("LD_LIBRARY_PATH", jlp);
+
+        LOG.info("Launching worker with command: {}. ", Utils.shellCmd(commandList));
+
+        String workerDir = ConfigUtils.workerRoot(_conf, _workerId);
+
+        launchWorkerProcess(commandList, topEnvironment, logPrefix, processExitCallback, new File(workerDir));
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java
new file mode 100644
index 0000000..7dacf14
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainerLauncher.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.storm.daemon.supervisor.Container.ContainerType;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.utils.LocalState;
+
+/**
+ * Launch containers with no security using standard java commands
+ */
+public class BasicContainerLauncher extends ContainerLauncher {
+    private final Map<String, Object> _conf;
+    private final String _supervisorId;
+    
+    public BasicContainerLauncher(Map<String, Object> conf, String supervisorId) throws IOException {
+        _conf = conf;
+        _supervisorId = supervisorId;
+    }
+
+    @Override
+    public Container launchContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+        Container container = new BasicContainer(ContainerType.LAUNCH, _conf, _supervisorId, port, assignment,
+                state, null);
+        container.setup();
+        container.launch();
+        return container;
+    }
+
+    @Override
+    public Container recoverContainer(int port, LocalAssignment assignment, LocalState state) throws IOException {
+        return new BasicContainer(ContainerType.RECOVER_FULL, _conf, _supervisorId, port, assignment,
+                state, null);
+    }
+
+    @Override
+    public Killable recoverContainer(String workerId, LocalState localState) throws IOException {
+        return new BasicContainer(ContainerType.RECOVER_PARTIAL, _conf, _supervisorId, -1, null,
+                    localState, workerId);
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
new file mode 100644
index 0000000..aea4b53
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/Container.java
@@ -0,0 +1,534 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Writer;
+import java.lang.ProcessBuilder.Redirect;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.storm.Config;
+import org.apache.storm.generated.LSWorkerHeartbeat;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+
+/**
+ * Represents a container that a worker will run in.
+ */
+public abstract class Container implements Killable {
+    private static final Logger LOG = LoggerFactory.getLogger(Container.class);
+    public static enum ContainerType {
+        LAUNCH(false, false),
+        RECOVER_FULL(true, false),
+        RECOVER_PARTIAL(true, true);
+
+        private final boolean _recovery;
+        private final boolean _onlyKillable;
+        
+        ContainerType(boolean recovery, boolean onlyKillable) {
+            _recovery = recovery;
+            _onlyKillable = onlyKillable;
+        }
+        
+        public boolean isRecovery() {
+            return _recovery;
+        }
+        
+        public void assertFull() {
+            if (_onlyKillable) {
+                throw new IllegalStateException("Container is only Killable.");
+            }
+        }
+        
+        public boolean isOnlyKillable() {
+            return _onlyKillable;
+        }
+    }
+    
+    protected final Map<String, Object> _conf;
+    protected final Map<String, Object> _topoConf; //Not set if RECOVER_PARTIAL
+    protected String _workerId; 
+    protected final String _topologyId; //Not set if RECOVER_PARTIAL
+    protected final String _supervisorId;
+    protected final int _port; //Not set if RECOVER_PARTIAL
+    protected final LocalAssignment _assignment; //Not set if RECOVER_PARTIAL
+    protected final AdvancedFSOps _ops;
+    protected ContainerType _type;
+    
+    /**
+     * Create a new Container.
+     * @param type the type of container being made.
+     * @param conf the supervisor config
+     * @param supervisorId the ID of the supervisor this is a part of.
+     * @param port the port the container is on.  Should be <= 0 if only a partial recovery
+     * @param assignment the assignment for this container. Should be null if only a partial recovery.
+     * @param workerId the id of the worker to use.  Must not be null if doing a partial recovery.
+     * @param topoConf the config of the topology (mostly for testing) if null 
+     * and not a partial recovery the real conf is read.
+     * @param ops file system operations (mostly for testing) if null a new one is made
+     * @throws IOException on any error.
+     */
+    protected Container(ContainerType type, Map<String, Object> conf, String supervisorId,
+            int port, LocalAssignment assignment,
+            String workerId, Map<String, Object> topoConf,  AdvancedFSOps ops) throws IOException {
+        assert(type != null);
+        assert(conf != null);
+        assert(supervisorId != null);
+        
+        if (ops == null) {
+            ops = AdvancedFSOps.make(conf);
+        }
+        
+        _workerId = workerId;
+        _type = type;
+        _port = port;
+        _ops = ops;
+        _conf = conf;
+        _supervisorId = supervisorId;
+        _assignment = assignment;
+        
+        if (_type.isOnlyKillable()) {
+            assert(_assignment == null);
+            assert(_port <= 0);
+            assert(_workerId != null);
+            _topologyId = null;
+            _topoConf = null;
+        } else {
+            assert(assignment != null);
+            assert(port > 0);
+            _topologyId = assignment.get_topology_id();
+            if (!_ops.doRequiredTopoFilesExist(_conf, _topologyId)) {
+                LOG.info("Missing topology storm code, so can't launch  worker with assignment {} for this supervisor {} on port {} with id {}", _assignment,
+                        _supervisorId, _port, _workerId);
+                throw new ContainerRecoveryException("Missing required topology files...");
+            }
+            if (topoConf == null) {
+                _topoConf = readTopoConf();
+            } else {
+                //For testing...
+                _topoConf = topoConf;
+            }
+        }
+    }
+    
+    @Override
+    public String toString() {
+        return "topo:" + _topologyId + " worker:" + _workerId;
+    }
+    
+    protected Map<String, Object> readTopoConf() throws IOException {
+        assert(_topologyId != null);
+        return ConfigUtils.readSupervisorStormConf(_conf, _topologyId);
+    }
+    
+    /**
+     * Kill a given process
+     * @param pid the id of the process to kill
+     * @throws IOException
+     */
+    protected void kill(long pid) throws IOException {
+        Utils.killProcessWithSigTerm(String.valueOf(pid));
+    }
+    
+    /**
+     * Kill a given process
+     * @param pid the id of the process to kill
+     * @throws IOException
+     */
+    protected void forceKill(long pid) throws IOException {
+        Utils.forceKillProcess(String.valueOf(pid));
+    }
+    
+    @Override
+    public void kill() throws IOException {
+        LOG.info("Killing {}:{}", _supervisorId, _workerId);
+        Set<Long> pids = getAllPids();
+
+        for (Long pid : pids) {
+            kill(pid);
+        }
+    }
+    
+    @Override
+    public void forceKill() throws IOException {
+        LOG.info("Force Killing {}:{}", _supervisorId, _workerId);
+        Set<Long> pids = getAllPids();
+        
+        for (Long pid : pids) {
+            forceKill(pid);
+        }
+    }
+    
+    /**
+     * Read the Heartbeat for the current container.
+     * @return the Heartbeat
+     * @throws IOException on any error
+     */
+    public LSWorkerHeartbeat readHeartbeat() throws IOException {
+        LocalState localState = ConfigUtils.workerState(_conf, _workerId);
+        LSWorkerHeartbeat hb = localState.getWorkerHeartBeat();
+        LOG.trace("{}: Reading heartbeat {}", _workerId, hb);
+        return hb;
+    }
+
+    /**
+     * Is a process alive and running?
+     * @param pid the PID of the running process
+     * @param user the user that is expected to own that process
+     * @return true if it is, else false
+     * @throws IOException on any error
+     */
+    protected boolean isProcessAlive(long pid, String user) throws IOException {
+        if (Utils.IS_ON_WINDOWS) {
+            return isWindowsProcessAlive(pid, user);
+        }
+        return isPosixProcessAlive(pid, user);
+    }
+    
+    private boolean isWindowsProcessAlive(long pid, String user) throws IOException {
+        boolean ret = false;
+        ProcessBuilder pb = new ProcessBuilder("tasklist", "/fo", "list", "/fi", "pid eq " + pid, "/v");
+        pb.redirectError(Redirect.INHERIT);
+        Process p = pb.start();
+        try (BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
+            String read;
+            while ((read = in.readLine()) != null) {
+                if (read.contains("User Name:")) { //Check for : in case someone called their user "User Name"
+                    //This line contains the user name for the pid we're looking up
+                    //Example line: "User Name:    exampleDomain\exampleUser"
+                    List<String> userNameLineSplitOnWhitespace = Arrays.asList(read.split(":"));
+                    if(userNameLineSplitOnWhitespace.size() == 2){
+                        List<String> userAndMaybeDomain = Arrays.asList(userNameLineSplitOnWhitespace.get(1).trim().split("\\\\"));
+                        String processUser = userAndMaybeDomain.size() == 2 ? userAndMaybeDomain.get(1) : userAndMaybeDomain.get(0);
+                        if(user.equals(processUser)){
+                            ret = true;
+                        } else {
+                            LOG.info("Found {} running as {}, but expected it to be {}", pid, processUser, user);
+                        }
+                    } else {
+                        LOG.error("Received unexpected output from tasklist command. Expected one colon in user name line. Line was {}", read);
+                    }
+                    break;
+                }
+            }
+        }
+        return ret;
+    }
+    
+    private boolean isPosixProcessAlive(long pid, String user) throws IOException {
+        boolean ret = false;
+        ProcessBuilder pb = new ProcessBuilder("ps", "-o", "user", "-p", String.valueOf(pid));
+        pb.redirectError(Redirect.INHERIT);
+        Process p = pb.start();
+        try (BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
+            String first = in.readLine();
+            assert("USER".equals(first));
+            String processUser;
+            while ((processUser = in.readLine()) != null) {
+                if (user.equals(processUser)) {
+                    ret = true;
+                    break;
+                } else {
+                    LOG.info("Found {} running as {}, but expected it to be {}", pid, processUser, user);
+                }
+            }
+        }
+        return ret;
+    }
+    
+    @Override
+    public boolean areAllProcessesDead() throws IOException {
+        Set<Long> pids = getAllPids();
+        String user = getWorkerUser();
+        
+        boolean allDead = true;
+        for (Long pid: pids) {
+            if (!isProcessAlive(pid, user)) {
+                LOG.debug("{}: PID {} is dead", _workerId, pid);
+            } else {
+                allDead = false;
+                break;
+            }
+        }
+        return allDead;
+    }
+
+    @Override
+    public void cleanUp() throws IOException {
+        cleanUpForRestart();
+    }
+    
+    /**
+     * Setup the container to run.  By default this creates the needed directories/links in the
+     * local file system
+     * PREREQUISITE: All needed blobs and topology, jars/configs have been downloaded and
+     * placed in the appropriate locations
+     * @throws IOException on any error
+     */
+    protected void setup() throws IOException {
+        _type.assertFull();
+        if (!_ops.doRequiredTopoFilesExist(_conf, _topologyId)) {
+            LOG.info("Missing topology storm code, so can't launch  worker with assignment {} for this supervisor {} on port {} with id {}", _assignment,
+                    _supervisorId, _port, _workerId);
+            throw new IllegalStateException("Not all needed files are here!!!!");
+        } 
+        LOG.info("Setting up {}:{}", _supervisorId, _workerId);
+
+        _ops.forceMkdir(new File(ConfigUtils.workerPidsRoot(_conf, _workerId)));
+        _ops.forceMkdir(new File(ConfigUtils.workerTmpRoot(_conf, _workerId)));
+        _ops.forceMkdir(new File(ConfigUtils.workerHeartbeatsRoot(_conf, _workerId)));
+        
+        File workerArtifacts = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port));
+        if (!_ops.fileExists(workerArtifacts)) {
+            _ops.forceMkdir(workerArtifacts);
+            _ops.setupStormCodeDir(_topoConf, workerArtifacts);
+        }
+    
+        String user = getWorkerUser();
+        writeLogMetadata(user);
+        saveWorkerUser(user);
+        createArtifactsLink();
+        createBlobstoreLinks();
+    }
+    
+    /**
+     * Write out the file used by the log viewer to allow/reject log access
+     * @param user the user this is going to run as
+     * @throws IOException on any error
+     */
+    @SuppressWarnings("unchecked")
+    protected void writeLogMetadata(String user) throws IOException {
+        _type.assertFull();
+        Map<String, Object> data = new HashMap<>();
+        data.put(Config.TOPOLOGY_SUBMITTER_USER, user);
+        data.put("worker-id", _workerId);
+
+        Set<String> logsGroups = new HashSet<>();
+        if (_topoConf.get(Config.LOGS_GROUPS) != null) {
+            List<String> groups = (List<String>) _topoConf.get(Config.LOGS_GROUPS);
+            for (String group : groups){
+                logsGroups.add(group);
+            }
+        }
+        if (_topoConf.get(Config.TOPOLOGY_GROUPS) != null) {
+            List<String> topGroups = (List<String>) _topoConf.get(Config.TOPOLOGY_GROUPS);
+            logsGroups.addAll(topGroups);
+        }
+        data.put(Config.LOGS_GROUPS, logsGroups.toArray());
+
+        Set<String> logsUsers = new HashSet<>();
+        if (_topoConf.get(Config.LOGS_USERS) != null) {
+            List<String> logUsers = (List<String>) _topoConf.get(Config.LOGS_USERS);
+            for (String logUser : logUsers){
+                logsUsers.add(logUser);
+            }
+        }
+        if (_topoConf.get(Config.TOPOLOGY_USERS) != null) {
+            List<String> topUsers = (List<String>) _topoConf.get(Config.TOPOLOGY_USERS);
+            for (String logUser : topUsers){
+                logsUsers.add(logUser);
+            }
+        }
+        data.put(Config.LOGS_USERS, logsUsers.toArray());
+
+        File file = ConfigUtils.getLogMetaDataFile(_conf, _topologyId, _port);
+
+        Yaml yaml = new Yaml();
+        try (Writer writer = _ops.getWriter(file)) {
+            yaml.dump(data, writer);
+        }
+    }
+    
+    /**
+     * Create symlink from the containers directory/artifacts to the artifacts directory
+     * @throws IOException on any error
+     */
+    protected void createArtifactsLink() throws IOException {
+        _type.assertFull();
+        File workerDir = new File(ConfigUtils.workerRoot(_conf, _workerId));
+        File topoDir = new File(ConfigUtils.workerArtifactsRoot(_conf, _topologyId, _port));
+        if (_ops.fileExists(workerDir)) {
+            LOG.debug("Creating symlinks for worker-id: {} topology-id: {} to its port artifacts directory", _workerId, _topologyId);
+            _ops.createSymlink(new File(workerDir, "artifacts"), topoDir);
+        }
+    }
+    
+    /**
+     * Create symlinks for each of the blobs from the container's directory to
+     * corresponding links in the storm dist directory.
+     * @throws IOException on any error.
+     */
+    protected void createBlobstoreLinks() throws IOException {
+        _type.assertFull();
+        String stormRoot = ConfigUtils.supervisorStormDistRoot(_conf, _topologyId);
+        String workerRoot = ConfigUtils.workerRoot(_conf, _workerId);
+        
+        @SuppressWarnings("unchecked")
+        Map<String, Map<String, Object>> blobstoreMap = (Map<String, Map<String, Object>>) _topoConf.get(Config.TOPOLOGY_BLOBSTORE_MAP);
+        List<String> blobFileNames = new ArrayList<>();
+        if (blobstoreMap != null) {
+            for (Map.Entry<String, Map<String, Object>> entry : blobstoreMap.entrySet()) {
+                String key = entry.getKey();
+                Map<String, Object> blobInfo = entry.getValue();
+                String ret = null;
+                if (blobInfo != null && blobInfo.containsKey("localname")) {
+                    ret = (String) blobInfo.get("localname");
+                } else {
+                    ret = key;
+                }
+                blobFileNames.add(ret);
+            }
+        }
+        List<String> resourceFileNames = new ArrayList<>();
+        resourceFileNames.add(ConfigUtils.RESOURCES_SUBDIR);
+        resourceFileNames.addAll(blobFileNames);
+        LOG.info("Creating symlinks for worker-id: {} storm-id: {} for files({}): {}", _workerId, _topologyId, resourceFileNames.size(), resourceFileNames);
+        _ops.createSymlink(new File(workerRoot, ConfigUtils.RESOURCES_SUBDIR), 
+                new File(stormRoot, ConfigUtils.RESOURCES_SUBDIR));
+        for (String fileName : blobFileNames) {
+            _ops.createSymlink(new File(workerRoot, fileName),
+                    new File(stormRoot, fileName));
+        }
+    }
+    
+    /**
+     * @return all of the pids that are a part of this container.
+     */
+    protected Set<Long> getAllPids() throws IOException {
+        Set<Long> ret = new HashSet<>();
+        for (String listing: Utils.readDirContents(ConfigUtils.workerPidsRoot(_conf, _workerId))) {
+            ret.add(Long.valueOf(listing));
+        }
+        
+        return ret;
+    }
+    
+    /** 
+     * @return the user that some operations should be done as.
+     * @throws IOException on any error
+     */
+    protected String getWorkerUser() throws IOException {
+        LOG.info("GET worker-user for {}", _workerId);
+        File file = new File(ConfigUtils.workerUserFile(_conf, _workerId));
+
+        if (_ops.fileExists(file)) {
+            return _ops.slurpString(file).trim();
+        } else if (_topoConf != null) { 
+            return (String) _topoConf.get(Config.TOPOLOGY_SUBMITTER_USER);
+        }
+        if (ConfigUtils.isLocalMode(_conf)) {
+            return System.getProperty("user.name");
+        } else {
+            File f = new File(ConfigUtils.workerArtifactsRoot(_conf));
+            if (f.exists()) {
+                return Files.getOwner(f.toPath()).getName();
+            }
+            throw new IllegalStateException("Could not recover the user for " + _workerId);
+        }
+    }
+    
+    protected void saveWorkerUser(String user) throws IOException {
+        _type.assertFull();
+        LOG.info("SET worker-user {} {}", _workerId, user);
+        _ops.dump(new File(ConfigUtils.workerUserFile(_conf, _workerId)), user);
+    }
+    
+    protected void deleteSavedWorkerUser() throws IOException {
+        LOG.info("REMOVE worker-user {}", _workerId);
+        _ops.deleteIfExists(new File(ConfigUtils.workerUserFile(_conf, _workerId)));
+    }
+    
+    /**
+     * Clean up the container partly preparing for restart.
+     * By default delete all of the temp directories we are going
+     * to get a new worker_id anyways.
+     * POST CONDITION: the workerId will be set to null
+     * @throws IOException on any error
+     */
+    public void cleanUpForRestart() throws IOException {
+        LOG.info("Cleaning up {}:{}", _supervisorId, _workerId);
+        Set<Long> pids = getAllPids();
+        String user = getWorkerUser();
+        
+        for (Long pid : pids) {
+            File path = new File(ConfigUtils.workerPidPath(_conf, _workerId, pid));
+            _ops.deleteIfExists(path, user, _workerId);
+        }
+        
+        //Always make sure to clean up everything else before worker directory
+        //is removed since that is what is going to trigger the retry for cleanup
+        _ops.deleteIfExists(new File(ConfigUtils.workerHeartbeatsRoot(_conf, _workerId)), user, _workerId);
+        _ops.deleteIfExists(new File(ConfigUtils.workerPidsRoot(_conf, _workerId)), user, _workerId);
+        _ops.deleteIfExists(new File(ConfigUtils.workerTmpRoot(_conf, _workerId)), user, _workerId);
+        _ops.deleteIfExists(new File(ConfigUtils.workerRoot(_conf, _workerId)), user, _workerId);
+        deleteSavedWorkerUser();
+        _workerId = null;
+    }
+    
+    /**
+     * Launch the process for the first time
+     * PREREQUISITE: setup has run and passed
+     * @throws IOException on any error
+     */
+    public abstract void launch() throws IOException;
+    
+    /**
+     * Restart the processes in this container
+     * PREREQUISITE: cleanUpForRestart has run and passed
+     * @throws IOException on any error
+     */
+    public abstract void relaunch() throws IOException;
+
+    /**
+     * @return true if the main process exited, else false. This is just best effort return false if unknown.
+     */
+    public abstract boolean didMainProcessExit();
+
+    /**
+     * Run a profiling request
+     * @param request the request to run
+     * @param stop is this a stop request?
+     * @return true if it succeeded, else false
+     * @throws IOException on any error
+     * @throws InterruptedException if running the command is interrupted.
+     */
+    public abstract boolean runProfiling(ProfileRequest request, boolean stop) throws IOException, InterruptedException;
+
+    /**
+     * @return the id of the container or null if there is no worker id right now.
+     */
+    public String getWorkerId() {
+        return _workerId;
+    }
+}


[02/21] storm git commit: STORM-2018: Supervisor V2

Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/integration/org/apache/storm/integration_test.clj b/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
index 238e0db..11e22fb 100644
--- a/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
+++ b/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
@@ -20,10 +20,14 @@
   (:import [org.apache.storm.generated InvalidTopologyException SubmitOptions TopologyInitialStatus RebalanceOptions])
   (:import [org.apache.storm.testing TestWordCounter TestWordSpout TestGlobalCount
             TestAggregatesCounter TestConfBolt AckFailMapTracker AckTracker TestPlannerSpout])
+  (:import [org.apache.storm.task WorkerTopologyContext])
   (:import [org.apache.storm.utils Time])
   (:import [org.apache.storm.tuple Fields])
+  (:import [org.mockito Mockito])
   (:use [org.apache.storm testing config clojure util])
   (:use [org.apache.storm.daemon common])
+  (:require [org.apache.storm [cluster :as cluster]])
+  (:require [org.apache.storm.daemon [executor :as executor]])
   (:require [org.apache.storm [thrift :as thrift]]))
 
 (deftest test-basic-topology
@@ -115,7 +119,6 @@
                              "timeout-tester"
                              {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10}
                              topology)
-      (advance-cluster-time cluster 11)
       (.feed feeder ["a"] 1)
       (.feed feeder ["b"] 2)
       (.feed feeder ["c"] 3)
@@ -282,7 +285,6 @@
                              "acking-test1"
                              {}
                              (:topology tracked))
-      (advance-cluster-time cluster 11)
       (.feed feeder1 [1])
       (tracked-wait tracked 1)
       (checker1 0)
@@ -325,7 +327,6 @@
                              "test-acking2"
                              {}
                              (:topology tracked))
-      (advance-cluster-time cluster 11)
       (.feed feeder [1])
       (tracked-wait tracked 1)
       (checker 0)
@@ -371,8 +372,7 @@
         {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10}
         topology
         (SubmitOptions. TopologyInitialStatus/INACTIVE))
-      (advance-cluster-time cluster 11)
-      (.feed feeder ["a"] 1)
+      (.feedNoWait feeder ["a"] 1)
       (advance-cluster-time cluster 9)
       (is (not @bolt-prepared?))
       (is (not @spout-opened?))        
@@ -396,7 +396,6 @@
                              "test"
                              {}
                              (:topology tracked))
-      (advance-cluster-time cluster 11)
       (.feed feeder [1])
       (tracked-wait tracked 1)
       (checker 1)
@@ -595,60 +594,42 @@
              (read-tuples results "2")
              )))))
 
-(defbolt report-errors-bolt {}
-  [tuple collector]
-  (doseq [i (range (.getValue tuple 0))]
-    (report-error! collector (RuntimeException.)))
-  (ack! collector tuple))
-
+;;This is more of a unit test, but getting the timing right for
+;; an integration test is really hard
 (deftest test-throttled-errors
   (with-simulated-time
-    (with-tracked-cluster [cluster]
-      (let [state (:storm-cluster-state cluster)
-            [feeder checker] (ack-tracking-feeder ["num"])
-            tracked (mk-tracked-topology
-                     cluster
-                     (topology
-                       {"1" (spout-spec feeder)}
-                       {"2" (bolt-spec {"1" :shuffle} report-errors-bolt)}))
-            _       (submit-local-topology (:nimbus cluster)
-                                             "test-errors"
-                                             {TOPOLOGY-ERROR-THROTTLE-INTERVAL-SECS 10
-                                              TOPOLOGY-MAX-ERROR-REPORT-PER-INTERVAL 4
-                                              TOPOLOGY-DEBUG true
-                                              }
-                                             (:topology tracked))
-            _ (advance-cluster-time cluster 11)
-            storm-id (get-storm-id state "test-errors")
-            errors-count (fn [] (count (.errors state storm-id "2")))]
-
-        (is (nil? (.last-error state storm-id "2")))
-
-        ;; so it launches the topology
-        (advance-cluster-time cluster 2)
-        (.feed feeder [6])
-        (tracked-wait tracked 1)
-        (is (= 4 (errors-count)))
-        (is (.last-error state storm-id "2"))
-        
-        (advance-time-secs! 5)
-        (.feed feeder [2])
-        (tracked-wait tracked 1)
-        (is (= 4 (errors-count)))
-        (is (.last-error state storm-id "2"))
-        
-        (advance-time-secs! 6)
-        (.feed feeder [2])
-        (tracked-wait tracked 1)
-        (is (= 6 (errors-count)))
-        (is (.last-error state storm-id "2"))
-        
-        (advance-time-secs! 6)
-        (.feed feeder [3])
-        (tracked-wait tracked 1)
-        (is (= 8 (errors-count)))
-        (is (.last-error state storm-id "2"))))))
-
+    (let [error-count (atom 0)
+          worker-context (Mockito/mock WorkerTopologyContext)
+          cluster-state
+             (reify cluster/StormClusterState
+               (report-error
+                 [this storm-id component-id node port error]
+                 (swap! error-count inc)))
+          error-fn (executor/throttled-report-error-fn 
+            {:storm-conf {TOPOLOGY-ERROR-THROTTLE-INTERVAL-SECS 10
+                          TOPOLOGY-MAX-ERROR-REPORT-PER-INTERVAL 4}
+             :storm-cluster-state cluster-state
+             :storm-id "topo"
+             :component-id "comp"
+             :worker-context worker-context})]
+        (. (Mockito/when (.getThisWorkerPort worker-context)) (thenReturn (Integer. 8080))) 
+        (error-fn (RuntimeException. "ERROR-1"))
+        (is (= 1 @error-count))
+        (error-fn (RuntimeException. "ERROR-2"))
+        (is (= 2 @error-count))
+        (error-fn (RuntimeException. "ERROR-3"))
+        (is (= 3 @error-count))
+        (error-fn (RuntimeException. "ERROR-4"))
+        (is (= 4 @error-count))
+        ;;Ignored
+        (error-fn (RuntimeException. "ERROR-5"))
+        (is (= 4 @error-count))
+        (Time/advanceTime 9000)
+        (error-fn (RuntimeException. "ERROR-6"))
+        (is (= 4 @error-count))
+        (Time/advanceTime 2000)
+        (error-fn (RuntimeException. "ERROR-7"))
+        (is (= 5 @error-count)))))
 
 (deftest test-acking-branching-complex
   ;; test acking with branching in the topology

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj b/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
index 5f71e86..f3955bb 100644
--- a/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
+++ b/storm-core/test/clj/integration/org/apache/storm/testing4j_test.clj
@@ -121,9 +121,9 @@
                              (it/agg-bolt 4))}))]
          (.submitTopology cluster
                           "test-acking2"
-                          (Config.)
+                          (doto (Config.)
+                            (.put TOPOLOGY-DEBUG true))
                           (.getTopology tracked))
-         (advance-cluster-time (.getState cluster) 11)
          (.feed feeder [1])
          (Testing/trackedWait tracked (int 1))
          (checker 0)
@@ -153,13 +153,14 @@
                             "timeout-tester"
                             storm-conf
                             topology)
+           ;;Wait for the topology to come up
            (.feed feeder ["a"] 1)
            (.feed feeder ["b"] 2)
            (.feed feeder ["c"] 3)
            (Testing/advanceClusterTime cluster (int 9))
            (it/assert-acked tracker 1 3)
            (is (not (.isFailed tracker 2)))
-           (Testing/advanceClusterTime cluster (int 12))
+           (Testing/advanceClusterTime cluster (int 20))
            (it/assert-failed tracker 2)
            ))))))
 
@@ -179,19 +180,21 @@
                            {"1" (thrift/mk-spout-spec feeder)}
                            {"2" (thrift/mk-bolt-spec {"1" :global} it/ack-every-other)})
                 storm-conf (doto (Config.)
+                             (.put TOPOLOGY-DEBUG true)
                              (.put TOPOLOGY-MESSAGE-TIMEOUT-SECS 10)
                              (.put TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS false))]
             (.submitTopology cluster
               "disable-timeout-tester"
               storm-conf
               topology)
+            ;;Wait for the topology to come up
             (.feed feeder ["a"] 1)
             (.feed feeder ["b"] 2)
             (.feed feeder ["c"] 3)
             (Testing/advanceClusterTime cluster (int 9))
             (it/assert-acked tracker 1 3)
             (is (not (.isFailed tracker 2)))
-            (Testing/advanceClusterTime cluster (int 12))
+            (Testing/advanceClusterTime cluster (int 20))
             (is (not (.isFailed tracker 2)))
             ))))))
 

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/org/apache/storm/logviewer_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/logviewer_test.clj b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
index 4d32a06..aae76e6 100644
--- a/storm-core/test/clj/org/apache/storm/logviewer_test.clj
+++ b/storm-core/test/clj/org/apache/storm/logviewer_test.clj
@@ -15,13 +15,16 @@
 ;; limitations under the License.
 (ns org.apache.storm.logviewer-test
   (:use [org.apache.storm config util])
-  (:require [org.apache.storm.daemon [logviewer :as logviewer]
-                                   [supervisor :as supervisor]])
+  (:require [org.apache.storm.daemon [logviewer :as logviewer]])
   (:require [conjure.core])
   (:use [clojure test])
   (:use [conjure core])
-  (:use [org.apache.storm.ui helpers])
-  (:import [org.apache.storm.daemon DirectoryCleaner])
+  (:use [org.apache.storm testing]
+        [org.apache.storm.ui helpers])
+  (:import [org.apache.storm.daemon DirectoryCleaner]
+           [org.apache.storm.daemon.supervisor SupervisorUtils]
+           [org.apache.storm.testing.staticmocking MockedSupervisorUtils]
+           [org.apache.storm.generated LSWorkerHeartbeat])
   (:import [java.nio.file Files Path DirectoryStream])
   (:import [java.nio.file Files])
   (:import [java.nio.file.attribute FileAttribute])
@@ -231,27 +234,33 @@
           mock-metaFile (mk-mock-File {:name "worker.yaml"
                                        :type :file})
           exp-id "id12345"
-          expected {exp-id port1-dir}]
-      (stubbing [supervisor/read-worker-heartbeats nil
-                 logviewer/get-metadata-file-for-wroker-logdir mock-metaFile
-                 logviewer/get-worker-id-from-metadata-file exp-id]
-        (is (= expected (logviewer/identify-worker-log-dirs [port1-dir])))))))
+          expected {exp-id port1-dir}
+          supervisor-util (Mockito/mock SupervisorUtils)]
+      (with-open [_ (MockedSupervisorUtils. supervisor-util)]
+        (stubbing [logviewer/get-metadata-file-for-wroker-logdir mock-metaFile
+                   logviewer/get-worker-id-from-metadata-file exp-id]
+          (. (Mockito/when (.readWorkerHeartbeatsImpl supervisor-util (Mockito/any))) (thenReturn nil))
+          (is (= expected (logviewer/identify-worker-log-dirs [port1-dir]))))))))
 
 (deftest test-get-dead-worker-dirs
-         (testing "return directories for workers that are not alive"
-                  (let [conf {SUPERVISOR-WORKER-TIMEOUT-SECS 5}
-                        id->hb {"42" {:time-secs 1}} ;; map for alive ids
-                        now-secs 2
-                        unexpected-dir1 (mk-mock-File {:name "dir1" :type :directory})
-                        expected-dir2 (mk-mock-File {:name "dir2" :type :directory})
-                        expected-dir3 (mk-mock-File {:name "dir3" :type :directory})
-                        log-dirs #{unexpected-dir1 expected-dir2 expected-dir3}]
-                       (stubbing [logviewer/identify-worker-log-dirs {"42" unexpected-dir1,
-                                                                      "007" expected-dir2,
-                                                                      "" expected-dir3} ;; this tests a directory with no yaml file thus no worker id
-                                  supervisor/read-worker-heartbeats id->hb]
-                                 (is (= #{expected-dir2 expected-dir3}
-                                        (logviewer/get-dead-worker-dirs conf now-secs log-dirs)))))))
+  (testing "return directories for workers that are not alive"
+    (let [conf {SUPERVISOR-WORKER-TIMEOUT-SECS 5}
+          hb (let [lwb (LSWorkerHeartbeat.)]
+                   (.set_time_secs lwb (int 1)) lwb)
+          id->hb {"42" hb}
+          now-secs 2
+          unexpected-dir1 (mk-mock-File {:name "dir1" :type :directory})
+          expected-dir2 (mk-mock-File {:name "dir2" :type :directory})
+          expected-dir3 (mk-mock-File {:name "dir3" :type :directory})
+          log-dirs #{unexpected-dir1 expected-dir2 expected-dir3}
+          supervisor-util (Mockito/mock SupervisorUtils)]
+      (with-open [_ (MockedSupervisorUtils. supervisor-util)]
+      (stubbing [logviewer/identify-worker-log-dirs {"42" unexpected-dir1,
+                                                     "007" expected-dir2,
+                                                     "" expected-dir3}] ;; this tests a directory with no yaml file thus no worker id
+        (. (Mockito/when (.readWorkerHeartbeatsImpl supervisor-util (Mockito/any))) (thenReturn id->hb))
+        (is (= #{expected-dir2 expected-dir3}
+              (logviewer/get-dead-worker-dirs conf now-secs log-dirs))))))))
 
 (deftest test-cleanup-fn
   (testing "cleanup function rmr's files of dead workers"

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/org/apache/storm/metrics_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/metrics_test.clj b/storm-core/test/clj/org/apache/storm/metrics_test.clj
index 9f051f6..ba48ea9 100644
--- a/storm-core/test/clj/org/apache/storm/metrics_test.clj
+++ b/storm-core/test/clj/org/apache/storm/metrics_test.clj
@@ -260,18 +260,18 @@
                     {"mybolt" (thrift/mk-bolt-spec {"myspout" :shuffle} ack-every-other)})]      
       (submit-local-topology (:nimbus cluster)
                              "metrics-tester"
-                             {}
+                             {TOPOLOGY-DEBUG true}
                              topology)
       
       (.feed feeder ["a"] 1)
       (advance-cluster-time cluster 6)
-      (assert-acked tracker 1)
       (assert-buckets! "myspout" "__fail-count/default" [] cluster)
       (assert-buckets! "myspout" "__ack-count/default" [1] cluster)
       (assert-buckets! "myspout" "__emit-count/default" [1] cluster)
       (assert-buckets! "myspout" "__transfer-count/default" [1] cluster)            
       (assert-buckets! "mybolt" "__ack-count/myspout:default" [1] cluster)     
       (assert-buckets! "mybolt" "__execute-count/myspout:default" [1] cluster)
+      (assert-acked tracker 1)
 
       (.feed feeder ["b"] 2)      
       (advance-cluster-time cluster 5)
@@ -312,18 +312,19 @@
                     {"mybolt" (thrift/mk-bolt-spec {"myspout" :global} ack-every-other)})]      
       (submit-local-topology (:nimbus cluster)
                              "timeout-tester"
-                             {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10}
+                             {TOPOLOGY-MESSAGE-TIMEOUT-SECS 10
+                              TOPOLOGY-DEBUG true}
                              topology)
       (.feed feeder ["a"] 1)
       (.feed feeder ["b"] 2)
       (.feed feeder ["c"] 3)
       (advance-cluster-time cluster 9)
-      (assert-acked tracker 1 3)
       (assert-buckets! "myspout" "__ack-count/default" [2] cluster)
       (assert-buckets! "myspout" "__emit-count/default" [3] cluster)
       (assert-buckets! "myspout" "__transfer-count/default" [3] cluster)
       (assert-buckets! "mybolt" "__ack-count/myspout:default" [2] cluster)
       (assert-buckets! "mybolt" "__execute-count/myspout:default" [3] cluster)
+      (assert-acked tracker 1 3)
       
       (is (not (.isFailed tracker 2)))
       (advance-cluster-time cluster 30)

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/org/apache/storm/nimbus_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/nimbus_test.clj b/storm-core/test/clj/org/apache/storm/nimbus_test.clj
index 7a1724d..c7b00b3 100644
--- a/storm-core/test/clj/org/apache/storm/nimbus_test.clj
+++ b/storm-core/test/clj/org/apache/storm/nimbus_test.clj
@@ -1056,6 +1056,14 @@
         (assert-files-in-dir [])
         ))))
 
+(defn wait-for-status [nimbus name status]
+  (while-timeout 5000
+    (let [topo-summary (first (filter (fn [topo] (= name (.get_name topo))) (.get_topologies (.getClusterInfo nimbus))))
+          topo-status (if topo-summary (.get_status topo-summary) "NOT-RUNNING")]
+      (log-message "WAITING FOR "name" TO BE " status " CURRENT " topo-status)
+      (not= topo-status status))
+    (Thread/sleep 100)))
+
 (deftest test-leadership
   "Tests that leader actions can only be performed by master and non leader fails to perform the same actions."
   (with-inprocess-zookeeper zk-port
@@ -1082,7 +1090,7 @@
               (submit-local-topology nimbus "t1" {} topology)
               ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
               (.rebalance nimbus "t1" (doto (RebalanceOptions.) (.set_wait_secs 0)))
-              (Thread/sleep 1000)
+              (wait-for-status nimbus "t1" "ACTIVE") 
               (.deactivate nimbus "t1")
               (.activate nimbus "t1")
               (.rebalance nimbus "t1" (RebalanceOptions.))

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj b/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
index d6a431f..c1eacd6 100644
--- a/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
+++ b/storm-core/test/clj/org/apache/storm/security/auth/drpc_auth_test.clj
@@ -98,42 +98,42 @@
 
 (defmacro with-simple-drpc-test-scenario
   [[strict? alice-client bob-client charlie-client alice-invok charlie-invok] & body]
-  (let [client-port (available-port)
-        invocations-port (available-port (inc client-port))
-        storm-conf (merge (read-storm-config)
-                          {DRPC-AUTHORIZER-ACL-STRICT strict?
+  `(let [client-port# (available-port)
+         invocations-port# (available-port (inc client-port#))
+         storm-conf# (merge (read-storm-config)
+                          {DRPC-AUTHORIZER-ACL-STRICT ~strict?
                            DRPC-AUTHORIZER-ACL-FILENAME "drpc-simple-acl-test-scenario.yaml"
                            STORM-THRIFT-TRANSPORT-PLUGIN "org.apache.storm.security.auth.digest.DigestSaslTransportPlugin"})]
-    `(with-server [~storm-conf
+    (with-server [storm-conf#
                    "org.apache.storm.security.auth.authorizer.DRPCSimpleACLAuthorizer"
                    "org.apache.storm.security.auth.digest.DigestSaslTransportPlugin"
                    "test/clj/org/apache/storm/security/auth/drpc-auth-server.jaas"
-                   ~client-port ~invocations-port]
+                   client-port# invocations-port#]
        (let [~alice-client (DRPCClient.
-                           (merge ~storm-conf {"java.security.auth.login.config"
+                           (merge storm-conf# {"java.security.auth.login.config"
                                               "test/clj/org/apache/storm/security/auth/drpc-auth-alice.jaas"})
                            "localhost"
-                           ~client-port)
+                           client-port#)
              ~bob-client (DRPCClient.
-                         (merge ~storm-conf {"java.security.auth.login.config"
+                         (merge storm-conf# {"java.security.auth.login.config"
                                             "test/clj/org/apache/storm/security/auth/drpc-auth-bob.jaas"})
                          "localhost"
-                         ~client-port)
+                         client-port#)
              ~charlie-client (DRPCClient.
-                               (merge ~storm-conf {"java.security.auth.login.config"
+                               (merge storm-conf# {"java.security.auth.login.config"
                                                   "test/clj/org/apache/storm/security/auth/drpc-auth-charlie.jaas"})
                                "localhost"
-                               ~client-port)
+                               client-port#)
              ~alice-invok (DRPCInvocationsClient.
-                            (merge ~storm-conf {"java.security.auth.login.config"
+                            (merge storm-conf# {"java.security.auth.login.config"
                                                "test/clj/org/apache/storm/security/auth/drpc-auth-alice.jaas"})
                             "localhost"
-                            ~invocations-port)
+                            invocations-port#)
              ~charlie-invok (DRPCInvocationsClient.
-                             (merge ~storm-conf {"java.security.auth.login.config"
+                             (merge storm-conf# {"java.security.auth.login.config"
                                                 "test/clj/org/apache/storm/security/auth/drpc-auth-charlie.jaas"})
                              "localhost"
-                             ~invocations-port)]
+                             invocations-port#)]
          (try
            ~@body
            (finally

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/clj/org/apache/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/supervisor_test.clj b/storm-core/test/clj/org/apache/storm/supervisor_test.clj
deleted file mode 100644
index 997de83..0000000
--- a/storm-core/test/clj/org/apache/storm/supervisor_test.clj
+++ /dev/null
@@ -1,751 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements.  See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership.  The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License.  You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.supervisor-test
-  (:use [clojure test])
-  (:require [conjure.core])
-  (:use [conjure core])
-  (:require [clojure.contrib [string :as contrib-str]])
-  (:require [clojure [string :as string] [set :as set]])
-  (:import [org.apache.storm.testing TestWordCounter TestWordSpout TestGlobalCount TestAggregatesCounter TestPlannerSpout])
-  (:import [org.apache.storm.scheduler ISupervisor])
-  (:import [org.apache.storm.generated RebalanceOptions])
-  (:import [java.util UUID])
-  (:import [java.io File])
-  (:import [java.nio.file Files])
-    (:import [java.nio.file.attribute FileAttribute] (org.apache.storm.topology TopologyBuilder))
-  (:use [org.apache.storm config testing util timer])
-  (:use [org.apache.storm.daemon common])
-  (:require [org.apache.storm.daemon [worker :as worker] [supervisor :as supervisor]]
-            [org.apache.storm [thrift :as thrift] [cluster :as cluster]])
-  (:use [conjure core])
-  (:require [clojure.java.io :as io]))
-
-(defn worker-assignment
-  "Return [storm-id executors]"
-  [cluster supervisor-id port]
-  (let [state (:storm-cluster-state cluster)
-        slot-assigns (for [storm-id (.assignments state nil)]
-                        (let [executors (-> (.assignment-info state storm-id nil)
-                                        :executor->node+port
-                                        reverse-map
-                                        (get [supervisor-id port] ))]
-                          (when executors [storm-id executors])
-                          ))
-        ret (find-first not-nil? slot-assigns)]
-    (when-not ret
-      (throw-runtime "Could not find assignment for worker"))
-    ret
-    ))
-
-(defn heartbeat-worker [supervisor port storm-id executors]
-  (let [conf (.get-conf supervisor)]
-    (worker/do-heartbeat {:conf conf
-                          :port port
-                          :storm-id storm-id
-                          :executors executors
-                          :worker-id (find-worker-id conf port)})))
-
-(defn heartbeat-workers [cluster supervisor-id ports]
-  (let [sup (get-supervisor cluster supervisor-id)]
-    (doseq [p ports]
-      (let [[storm-id executors] (worker-assignment cluster supervisor-id p)]
-        (heartbeat-worker sup p storm-id executors)
-        ))))
-
-(defn validate-launched-once [launched supervisor->ports storm-id]
-  (let [counts (map count (vals launched))
-        launched-supervisor->ports (apply merge-with set/union
-                                          (for [[[s p] sids] launched
-                                                :when (some #(= % storm-id) sids)]
-                                            {s #{p}}))
-        supervisor->ports (map-val set supervisor->ports)]
-    (is (every? (partial = 1) counts))
-    (is (= launched-supervisor->ports supervisor->ports))
-    ))
-
-(deftest launches-assignment
-  (with-simulated-time-local-cluster [cluster :supervisors 0
-    :daemon-conf {NIMBUS-DO-NOT-REASSIGN true
-                  SUPERVISOR-WORKER-START-TIMEOUT-SECS 5
-                  SUPERVISOR-WORKER-TIMEOUT-SECS 15
-                  SUPERVISOR-MONITOR-FREQUENCY-SECS 3}]
-    (letlocals
-      (bind topology (thrift/mk-topology
-                       {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 4)}
-                       {}))
-      (bind sup1 (add-supervisor cluster :id "sup1" :ports [1 2 3 4]))
-      (bind changed (capture-changed-workers
-                      (submit-mocked-assignment
-                        (:nimbus cluster)
-                        (:storm-cluster-state cluster)
-                        "test"
-                        {TOPOLOGY-WORKERS 3}
-                        topology
-                        {1 "1"
-                         2 "1"
-                         3 "1"
-                         4 "1"}
-                        {[1 1] ["sup1" 1]
-                         [2 2] ["sup1" 2]
-                         [3 3] ["sup1" 3]
-                         [4 4] ["sup1" 3]}
-                        {["sup1" 1] [0.0 0.0 0.0]
-                         ["sup1" 2] [0.0 0.0 0.0]
-                         ["sup1" 3] [0.0 0.0 0.0]
-                         })
-                      ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
-                      (.rebalance (:nimbus cluster) "test" (doto (RebalanceOptions.) (.set_wait_secs 0)))
-                      (advance-cluster-time cluster 2)
-                      (heartbeat-workers cluster "sup1" [1 2 3])
-                      (advance-cluster-time cluster 10)))
-      (bind storm-id (get-storm-id (:storm-cluster-state cluster) "test"))
-      (is (empty? (:shutdown changed)))
-      (validate-launched-once (:launched changed) {"sup1" [1 2 3]} storm-id)
-      (bind changed (capture-changed-workers
-                        (doseq [i (range 10)]
-                          (heartbeat-workers cluster "sup1" [1 2 3])
-                          (advance-cluster-time cluster 10))
-                        ))
-      (is (empty? (:shutdown changed)))
-      (is (empty? (:launched changed)))
-      (bind changed (capture-changed-workers
-                      (heartbeat-workers cluster "sup1" [1 2])
-                      (advance-cluster-time cluster 10)
-                      ))
-      (validate-launched-once (:launched changed) {"sup1" [3]} storm-id)
-      (is (= {["sup1" 3] 1} (:shutdown changed)))
-      )))
-
-(deftest test-multiple-active-storms-multiple-supervisors
-  (with-simulated-time-local-cluster [cluster :supervisors 0
-    :daemon-conf {NIMBUS-DO-NOT-REASSIGN true
-                  SUPERVISOR-WORKER-START-TIMEOUT-SECS 5
-                  SUPERVISOR-WORKER-TIMEOUT-SECS 15
-                  SUPERVISOR-MONITOR-FREQUENCY-SECS 3}]
-    (letlocals
-      (bind topology (thrift/mk-topology
-                       {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 4)}
-                       {}))
-      (bind topology2 (thrift/mk-topology
-                       {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 3)}
-                       {}))
-      (bind sup1 (add-supervisor cluster :id "sup1" :ports [1 2 3 4]))
-      (bind sup2 (add-supervisor cluster :id "sup2" :ports [1 2]))
-      (bind changed (capture-changed-workers
-                      (submit-mocked-assignment
-                        (:nimbus cluster)
-                        (:storm-cluster-state cluster)
-                        "test"
-                        {TOPOLOGY-WORKERS 3 TOPOLOGY-MESSAGE-TIMEOUT-SECS 40}
-                        topology
-                        {1 "1"
-                         2 "1"
-                         3 "1"
-                         4 "1"}
-                        {[1 1] ["sup1" 1]
-                         [2 2] ["sup1" 2]
-                         [3 3] ["sup2" 1]
-                         [4 4] ["sup2" 1]}
-                        {["sup1" 1] [0.0 0.0 0.0]
-                         ["sup1" 2] [0.0 0.0 0.0]
-                         ["sup2" 1] [0.0 0.0 0.0]
-                         })
-                      ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
-                      (.rebalance (:nimbus cluster) "test" (doto (RebalanceOptions.) (.set_wait_secs 0)))
-                      (advance-cluster-time cluster 2)
-                      (heartbeat-workers cluster "sup1" [1 2])
-                      (heartbeat-workers cluster "sup2" [1])
-                      ))
-      (bind storm-id (get-storm-id (:storm-cluster-state cluster) "test"))
-      (is (empty? (:shutdown changed)))
-      (validate-launched-once (:launched changed) {"sup1" [1 2] "sup2" [1]} storm-id)
-      (bind changed (capture-changed-workers
-                      (submit-mocked-assignment
-                        (:nimbus cluster)
-                        (:storm-cluster-state cluster)
-                        "test2"
-                        {TOPOLOGY-WORKERS 2}
-                        topology2
-                        {1 "1"
-                         2 "1"
-                         3 "1"}
-                        {[1 1] ["sup1" 3]
-                         [2 2] ["sup1" 3]
-                         [3 3] ["sup2" 2]}
-                        {["sup1" 3] [0.0 0.0 0.0]
-                         ["sup2" 2] [0.0 0.0 0.0]
-                         })
-                      ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
-                      (.rebalance (:nimbus cluster) "test2" (doto (RebalanceOptions.) (.set_wait_secs 0)))
-                      (advance-cluster-time cluster 2)
-                      (heartbeat-workers cluster "sup1" [3])
-                      (heartbeat-workers cluster "sup2" [2])
-                      ))
-      (bind storm-id2 (get-storm-id (:storm-cluster-state cluster) "test2"))
-      (is (empty? (:shutdown changed)))
-      (validate-launched-once (:launched changed) {"sup1" [3] "sup2" [2]} storm-id2)
-      (bind changed (capture-changed-workers
-        (.killTopology (:nimbus cluster) "test")
-        (doseq [i (range 4)]
-          (advance-cluster-time cluster 8)
-          (heartbeat-workers cluster "sup1" [1 2 3])
-          (heartbeat-workers cluster "sup2" [1 2])
-          )))
-      (is (empty? (:shutdown changed)))
-      (is (empty? (:launched changed)))
-      (bind changed (capture-changed-workers
-        (advance-cluster-time cluster 12)
-        ))
-      (is (empty? (:launched changed)))
-      (is (= {["sup1" 1] 1 ["sup1" 2] 1 ["sup2" 1] 1} (:shutdown changed)))
-      (bind changed (capture-changed-workers
-        (doseq [i (range 10)]
-          (heartbeat-workers cluster "sup1" [3])
-          (heartbeat-workers cluster "sup2" [2])
-          (advance-cluster-time cluster 10)
-          )))
-      (is (empty? (:shutdown changed)))
-      (is (empty? (:launched changed)))
-      ;; TODO check that downloaded code is cleaned up only for the one storm
-      )))
-
-(defn get-heartbeat [cluster supervisor-id]
-  (.supervisor-info (:storm-cluster-state cluster) supervisor-id))
-
-(defn check-heartbeat [cluster supervisor-id within-secs]
-  (let [hb (get-heartbeat cluster supervisor-id)
-        time-secs (:time-secs hb)
-        now (current-time-secs)
-        delta (- now time-secs)]
-    (is (>= delta 0))
-    (is (<= delta within-secs))
-    ))
-
-(deftest heartbeats-to-nimbus
-  (with-simulated-time-local-cluster [cluster :supervisors 0
-    :daemon-conf {SUPERVISOR-WORKER-START-TIMEOUT-SECS 15
-                  SUPERVISOR-HEARTBEAT-FREQUENCY-SECS 3}]
-    (letlocals
-      (bind sup1 (add-supervisor cluster :id "sup" :ports [5 6 7]))
-      (advance-cluster-time cluster 4)
-      (bind hb (get-heartbeat cluster "sup"))
-      (is (= #{5 6 7} (set (:meta hb))))
-      (check-heartbeat cluster "sup" 3)
-      (advance-cluster-time cluster 3)
-      (check-heartbeat cluster "sup" 3)
-      (advance-cluster-time cluster 3)
-      (check-heartbeat cluster "sup" 3)
-      (advance-cluster-time cluster 15)
-      (check-heartbeat cluster "sup" 3)
-      (bind topology (thrift/mk-topology
-                       {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 4)}
-                       {}))
-      ;; prevent them from launching by capturing them
-      (capture-changed-workers
-       (submit-local-topology (:nimbus cluster) "test" {TOPOLOGY-WORKERS 2} topology)
-       (advance-cluster-time cluster 3)
-       (check-heartbeat cluster "sup" 3)
-       (advance-cluster-time cluster 3)
-       (check-heartbeat cluster "sup" 3)
-       (advance-cluster-time cluster 3)
-       (check-heartbeat cluster "sup" 3)
-       (advance-cluster-time cluster 20)
-       (check-heartbeat cluster "sup" 3))
-      )))
-
-(deftest test-worker-launch-command
-  (testing "*.worker.childopts configuration"
-    (let [mock-port "42"
-          mock-storm-id "fake-storm-id"
-          mock-worker-id "fake-worker-id"
-          mock-mem-onheap 512
-          mock-cp (str file-path-separator "base" class-path-separator file-path-separator "stormjar.jar")
-          mock-sensitivity "S3"
-          mock-cp "/base:/stormjar.jar"
-          builder (TopologyBuilder.)
-          \u2028_ (.setSpout builder "wordSpout" (TestWordSpout.) 1)
-          \u2028_ (.shuffleGrouping (.setBolt builder "wordCountBolt" (TestWordCounter.) 1) "wordSpout")
-          \u2028mock-storm-topology (.createTopology builder)
-          exp-args-fn (fn [opts topo-opts classpath]
-                       (concat [(supervisor/java-cmd) "-cp" classpath
-                               (str "-Dlogfile.name=" "worker.log")
-                               "-Dstorm.home="
-                               (str "-Dworkers.artifacts=" "/tmp/workers-artifacts")
-                               (str "-Dstorm.id=" mock-storm-id)
-                               (str "-Dworker.id=" mock-worker-id)
-                               (str "-Dworker.port=" mock-port)
-                               "-Dstorm.log.dir=/logs"
-                               "-Dlog4j.configurationFile=/log4j2/worker.xml"
-                               "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector"
-                               "org.apache.storm.LogWriter"]
-                               [(supervisor/java-cmd) "-server"]
-                               opts
-                               topo-opts
-                               ["-Djava.library.path="
-                                (str "-Dlogfile.name=" "worker.log")
-                                "-Dstorm.home="
-                                "-Dworkers.artifacts=/tmp/workers-artifacts"
-                                "-Dstorm.conf.file="
-                                "-Dstorm.options="
-                                (str "-Dstorm.log.dir=" file-path-separator "logs")
-                                (str "-Djava.io.tmpdir=" file-path-separator "storm-local" file-path-separator
-                                     "workers" file-path-separator mock-worker-id file-path-separator "tmp")
-                                (str "-Dlogging.sensitivity=" mock-sensitivity)
-                                (str "-Dlog4j.configurationFile=" file-path-separator "log4j2" file-path-separator "worker.xml")
-                                "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector"
-                                (str "-Dstorm.id=" mock-storm-id)
-                                (str "-Dworker.id=" mock-worker-id)
-                                (str "-Dworker.port=" mock-port)
-                                "-cp" classpath
-                                "org.apache.storm.daemon.worker"
-                                mock-storm-id
-                                mock-port
-                                mock-worker-id]))]
-      (testing "testing *.worker.childopts as strings with extra spaces"
-        (let [string-opts "-Dfoo=bar  -Xmx1024m"
-              topo-string-opts "-Dkau=aux   -Xmx2048m"
-              exp-args (exp-args-fn ["-Dfoo=bar" "-Xmx1024m"]
-                                    ["-Dkau=aux" "-Xmx2048m"]
-                                    mock-cp)
-              mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
-                                      WORKER-CHILDOPTS string-opts}}]
-          (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
-                                                   topo-string-opts}
-                     read-supervisor-topology mock-storm-topology
-                     add-to-classpath mock-cp
-                     supervisor-stormdist-root nil
-                     launch-process nil
-                     set-worker-user! nil
-                     supervisor/jlp nil
-                     worker-artifacts-root "/tmp/workers-artifacts"
-                     supervisor/write-log-metadata! nil
-                     supervisor/create-blobstore-links nil]
-            (supervisor/launch-worker mock-supervisor
-                                      mock-storm-id
-                                      mock-port
-                                      mock-worker-id
-                                      mock-mem-onheap)
-            (verify-first-call-args-for-indices launch-process
-                                                [0]
-                                                exp-args))))
-      (testing "testing *.worker.childopts as list of strings, with spaces in values"
-        (let [list-opts '("-Dopt1='this has a space in it'" "-Xmx1024m")
-              topo-list-opts '("-Dopt2='val with spaces'" "-Xmx2048m")
-              exp-args (exp-args-fn list-opts topo-list-opts mock-cp)
-              mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
-                                      WORKER-CHILDOPTS list-opts}}]
-          (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
-                                                   topo-list-opts}
-                     read-supervisor-topology mock-storm-topology
-                     add-to-classpath mock-cp
-                     supervisor-stormdist-root nil
-                     launch-process nil
-                     set-worker-user! nil
-                     supervisor/jlp nil
-                     supervisor/write-log-metadata! nil
-                     supervisor/create-blobstore-links nil
-                     worker-artifacts-root "/tmp/workers-artifacts"]
-            (supervisor/launch-worker mock-supervisor
-                                      mock-storm-id
-                                      mock-port
-                                      mock-worker-id
-                                      mock-mem-onheap)
-            (verify-first-call-args-for-indices launch-process
-                                                [0]
-                                                exp-args))))
-      (testing "testing topology.classpath is added to classpath"
-        (let [topo-cp (str file-path-separator "any" file-path-separator "path")
-              exp-args (exp-args-fn [] [] (add-to-classpath mock-cp [topo-cp]))
-              mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed}}]
-          (stubbing [read-supervisor-storm-conf {TOPOLOGY-CLASSPATH topo-cp}
-                     supervisor-stormdist-root nil
-                     read-supervisor-topology mock-storm-topology
-                     supervisor/jlp nil
-                     worker-artifacts-root "/tmp/workers-artifacts"
-                     set-worker-user! nil
-                     supervisor/write-log-metadata! nil
-                     launch-process nil
-                     current-classpath (str file-path-separator "base")
-                     supervisor/create-blobstore-links nil]
-                    (supervisor/launch-worker mock-supervisor
-                                              mock-storm-id
-                                              mock-port
-                                              mock-worker-id
-                                              mock-mem-onheap)
-                    (verify-first-call-args-for-indices launch-process
-                                                        [0]
-                                                        exp-args))))
-      (testing "testing topology.environment is added to environment for worker launch"
-        (let [topo-env {"THISVAR" "somevalue" "THATVAR" "someothervalue"}
-              full-env (merge topo-env {"LD_LIBRARY_PATH" nil})
-              exp-args (exp-args-fn [] [] mock-cp)
-              mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed}}]
-          (stubbing [read-supervisor-storm-conf {TOPOLOGY-ENVIRONMENT topo-env}
-                     supervisor-stormdist-root nil
-                     read-supervisor-topology mock-storm-topology
-                     supervisor/jlp nil
-                     worker-artifacts-root "/tmp/workers-artifacts"
-                     launch-process nil
-                     set-worker-user! nil
-                     supervisor/write-log-metadata! nil
-                     current-classpath (str file-path-separator "base")
-                     supervisor/create-blobstore-links nil]
-                    (supervisor/launch-worker mock-supervisor
-                                              mock-storm-id
-                                              mock-port
-                                              mock-worker-id
-                                              mock-mem-onheap)
-                    (verify-first-call-args-for-indices launch-process
-                                                        [2]
-                                                        full-env)))))))
-
-(deftest test-worker-launch-command-run-as-user
-  (testing "*.worker.childopts configuration"
-    (let [mock-port "42"
-          mock-storm-id "fake-storm-id"
-          mock-worker-id "fake-worker-id"
-          mock-mem-onheap 512
-          mock-sensitivity "S3"
-          mock-cp "mock-classpath'quote-on-purpose"
-          builder (TopologyBuilder.)
-          \u2028_ (.setSpout builder "wordSpout" (TestWordSpout.) 1)
-          \u2028_ (.shuffleGrouping (.setBolt builder "wordCountBolt" (TestWordCounter.) 1) "wordSpout")
-          \u2028mock-storm-topology (.createTopology builder)
-          attrs (make-array FileAttribute 0)
-          storm-local (.getCanonicalPath (.toFile (Files/createTempDirectory "storm-local" attrs)))
-          worker-script (str storm-local "/workers/" mock-worker-id "/storm-worker-script.sh")
-          exp-launch ["/bin/worker-launcher"
-                      "me"
-                      "worker"
-                      (str storm-local "/workers/" mock-worker-id)
-                      worker-script]
-          exp-script-fn (fn [opts topo-opts]
-                          (str "#!/bin/bash\n'export' 'LD_LIBRARY_PATH=';\n\nexec 'java'"
-                               " '-cp' 'mock-classpath'\"'\"'quote-on-purpose'"
-                               " '-Dlogfile.name=" "worker.log'"
-                               " '-Dstorm.home='"
-                               " '-Dworkers.artifacts=" (str storm-local "/workers-artifacts'")
-                               " '-Dstorm.id=" mock-storm-id "'"
-                               " '-Dworker.id=" mock-worker-id "'"
-                               " '-Dworker.port=" mock-port "'"
-                               " '-Dstorm.log.dir=/logs'"
-                               " '-Dlog4j.configurationFile=/log4j2/worker.xml'"
-                               " '-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector'"
-                               " 'org.apache.storm.LogWriter'"
-                               " 'java' '-server'"
-                               " " (shell-cmd opts)
-                               " " (shell-cmd topo-opts)
-                               " '-Djava.library.path='"
-                               " '-Dlogfile.name=" "worker.log'"
-                               " '-Dstorm.home='"
-                               " '-Dworkers.artifacts=" (str storm-local "/workers-artifacts'")
-                               " '-Dstorm.conf.file='"
-                               " '-Dstorm.options='"
-                               " '-Dstorm.log.dir=/logs'"
-                               " '-Djava.io.tmpdir=" (str storm-local "/workers/" mock-worker-id "/tmp'")
-                               " '-Dlogging.sensitivity=" mock-sensitivity "'"
-                               " '-Dlog4j.configurationFile=/log4j2/worker.xml'"
-                               " '-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector'"
-                               " '-Dstorm.id=" mock-storm-id "'"
-                               " '-Dworker.id=" mock-worker-id "'"
-                               " '-Dworker.port=" mock-port "'"
-                               " '-cp' 'mock-classpath'\"'\"'quote-on-purpose'"
-                               " 'org.apache.storm.daemon.worker'"
-                               " '" mock-storm-id "'"
-                               " '" mock-port "'"
-                               " '" mock-worker-id "';"))]
-      (try
-        (testing "testing *.worker.childopts as strings with extra spaces"
-          (let [string-opts "-Dfoo=bar  -Xmx1024m"
-                topo-string-opts "-Dkau=aux   -Xmx2048m"
-                exp-script (exp-script-fn ["-Dfoo=bar" "-Xmx1024m"]
-                                          ["-Dkau=aux" "-Xmx2048m"])
-                _ (.mkdirs (io/file storm-local "workers" mock-worker-id))
-                mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
-                                        STORM-LOCAL-DIR storm-local
-                                        STORM-WORKERS-ARTIFACTS-DIR (str storm-local "/workers-artifacts")
-                                        SUPERVISOR-RUN-WORKER-AS-USER true
-                                        WORKER-CHILDOPTS string-opts}}]
-            (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
-                                                   topo-string-opts
-                                                   TOPOLOGY-SUBMITTER-USER "me"}
-                       read-supervisor-topology mock-storm-topology
-                       add-to-classpath mock-cp
-                       supervisor-stormdist-root nil
-                       launch-process nil
-                       set-worker-user! nil
-                       supervisor/java-cmd "java"
-                       supervisor/jlp nil
-                       supervisor/write-log-metadata! nil]
-                      (supervisor/launch-worker mock-supervisor
-                                                mock-storm-id
-                                                mock-port
-                                                mock-worker-id
-                                                mock-mem-onheap)
-                      (verify-first-call-args-for-indices launch-process
-                                                          [0]
-                                                          exp-launch))
-            (is (= (slurp worker-script) exp-script))))
-        (finally (rmr storm-local)))
-      (.mkdirs (io/file storm-local "workers" mock-worker-id))
-      (try
-        (testing "testing *.worker.childopts as list of strings, with spaces in values"
-          (let [list-opts '("-Dopt1='this has a space in it'" "-Xmx1024m")
-                topo-list-opts '("-Dopt2='val with spaces'" "-Xmx2048m")
-                exp-script (exp-script-fn list-opts topo-list-opts)
-                mock-supervisor {:conf {STORM-CLUSTER-MODE :distributed
-                                        STORM-LOCAL-DIR storm-local
-                                        STORM-WORKERS-ARTIFACTS-DIR (str storm-local "/workers-artifacts")
-                                        SUPERVISOR-RUN-WORKER-AS-USER true
-                                        WORKER-CHILDOPTS list-opts}}]
-            (stubbing [read-supervisor-storm-conf {TOPOLOGY-WORKER-CHILDOPTS
-                                                   topo-list-opts
-                                                   TOPOLOGY-SUBMITTER-USER "me"}
-                       read-supervisor-topology mock-storm-topology
-                       add-to-classpath mock-cp
-                       supervisor-stormdist-root nil
-                       launch-process nil
-                       set-worker-user! nil
-                       supervisor/java-cmd "java"
-                       supervisor/jlp nil
-                       supervisor/write-log-metadata! nil]
-                      (supervisor/launch-worker mock-supervisor
-                                                mock-storm-id
-                                                mock-port
-                                                mock-worker-id
-                                                mock-mem-onheap)
-                      (verify-first-call-args-for-indices launch-process
-                                                          [0]
-                                                          exp-launch))
-            (is (= (slurp worker-script) exp-script))))
-        (finally (rmr storm-local))))))
-
-(deftest test-workers-go-bananas
-  ;; test that multiple workers are started for a port, and test that
-  ;; supervisor shuts down propertly (doesn't shutdown the most
-  ;; recently launched one, checks heartbeats correctly, etc.)
-  )
-
-(deftest downloads-code
-  )
-
-(deftest test-stateless
-  )
-
-(deftest cleans-up-on-unassign
-  ;; TODO just do reassign, and check that cleans up worker states after killing but doesn't get rid of downloaded code
-  )
-
-(deftest test-supervisor-data-acls
-  (testing "supervisor-data uses correct ACLs"
-    (let [scheme "digest"
-          digest "storm:thisisapoorpassword"
-          auth-conf {STORM-ZOOKEEPER-AUTH-SCHEME scheme
-                     STORM-ZOOKEEPER-AUTH-PAYLOAD digest}
-          expected-acls supervisor/SUPERVISOR-ZK-ACLS
-          fake-isupervisor (reify ISupervisor
-                             (getSupervisorId [this] nil)
-                             (getAssignmentId [this] nil))]
-      (stubbing [uptime-computer nil
-                 cluster/mk-storm-cluster-state nil
-                 supervisor-state nil
-                 local-hostname nil
-                 mk-timer nil
-                 supervisor-local-dir nil]
-        (supervisor/supervisor-data auth-conf nil fake-isupervisor)
-        (verify-call-times-for cluster/mk-storm-cluster-state 1)
-        (verify-first-call-args-for-indices cluster/mk-storm-cluster-state [2]
-                                            expected-acls)))))
-
-(deftest test-write-log-metadata
-  (testing "supervisor writes correct data to logs metadata file"
-    (let [exp-owner "alice"
-          exp-worker-id "42"
-          exp-storm-id "0123456789"
-          exp-port 4242
-          exp-logs-users ["bob" "charlie" "daryl"]
-          exp-logs-groups ["read-only-group" "special-group"]
-          storm-conf {TOPOLOGY-SUBMITTER-USER "alice"
-                      TOPOLOGY-USERS ["charlie" "bob"]
-                      TOPOLOGY-GROUPS ["special-group"]
-                      LOGS-GROUPS ["read-only-group"]
-                      LOGS-USERS ["daryl"]}
-          exp-data {TOPOLOGY-SUBMITTER-USER exp-owner
-                    "worker-id" exp-worker-id
-                    LOGS-USERS exp-logs-users
-                    LOGS-GROUPS exp-logs-groups}
-          conf {}]
-      (mocking [supervisor/write-log-metadata-to-yaml-file!]
-        (supervisor/write-log-metadata! storm-conf exp-owner exp-worker-id
-                                        exp-storm-id exp-port conf)
-        (verify-called-once-with-args supervisor/write-log-metadata-to-yaml-file!
-                                      exp-storm-id exp-port exp-data conf)))))
-
-(deftest test-worker-launcher-requires-user
-  (testing "worker-launcher throws on blank user"
-    (mocking [launch-process]
-      (is (thrown-cause-with-msg? java.lang.IllegalArgumentException
-                                  #"(?i).*user cannot be blank.*"
-                                  (supervisor/worker-launcher {} nil ""))))))
-
-(defn found? [sub-str input-str]
-  (if (string? input-str)
-    (contrib-str/substring? sub-str (str input-str))
-    (boolean (some #(contrib-str/substring? sub-str %) input-str))))
-
-(defn not-found? [sub-str input-str]
-    (complement (found? sub-str input-str)))
-
-(deftest test-substitute-childopts-happy-path-string
-  (testing "worker-launcher replaces ids in childopts"
-    (let [worker-id "w-01"
-          topology-id "s-01"
-          port 9999
-          mem-onheap 512
-          childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m -Xmx%HEAP-MEM%m"
-          expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m" "-Xmx512m")
-          childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
-      (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-happy-path-list
-  (testing "worker-launcher replaces ids in childopts"
-    (let [worker-id "w-01"
-          topology-id "s-01"
-          port 9999
-          mem-onheap 512
-          childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m" "-Xmx%HEAP-MEM%m")
-          expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m" "-Xmx512m")
-          childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
-      (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-happy-path-list-arraylist
-  (testing "worker-launcher replaces ids in childopts"
-    (let [worker-id "w-01"
-          topology-id "s-01"
-          port 9999
-          mem-onheap 512
-          childopts '["-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log" "-Xms256m" "-Xmx%HEAP-MEM%m"]
-          expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01-w-01-9999.log" "-Xms256m" "-Xmx512m")
-          childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
-      (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-topology-id-alone
-  (testing "worker-launcher replaces ids in childopts"
-    (let [worker-id "w-01"
-          topology-id "s-01"
-          port 9999
-          mem-onheap 512
-          childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%TOPOLOGY-ID%.log"
-          expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-s-01.log")
-          childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
-      (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-no-keys
-  (testing "worker-launcher has no ids to replace in childopts"
-    (let [worker-id "w-01"
-          topology-id "s-01"
-          port 9999
-          mem-onheap 512
-          childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log"
-          expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker.log")
-          childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
-      (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-nil-childopts
-  (testing "worker-launcher has nil childopts"
-    (let [worker-id "w-01"
-          topology-id "s-01"
-          port 9999
-          mem-onheap 512
-          childopts nil
-          expected-childopts nil
-          childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
-      (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-substitute-childopts-nil-ids
-  (testing "worker-launcher has nil ids"
-    (let [worker-id nil
-          topology-id "s-01"
-          port 9999
-          mem-onheap 512
-          childopts "-Xloggc:/home/y/lib/storm/current/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log"
-          expected-childopts '("-Xloggc:/home/y/lib/storm/current/logs/gc.worker-9999-s-01--9999.log")
-          childopts-with-ids (supervisor/substitute-childopts childopts worker-id topology-id port mem-onheap)]
-      (is (= expected-childopts childopts-with-ids)))))
-
-(deftest test-retry-read-assignments
-  (with-simulated-time-local-cluster [cluster
-                                      :supervisors 0
-                                      :ports-per-supervisor 2
-                                      :daemon-conf {NIMBUS-DO-NOT-REASSIGN true
-                                                    NIMBUS-MONITOR-FREQ-SECS 10
-                                                    TOPOLOGY-MESSAGE-TIMEOUT-SECS 30
-                                                    TOPOLOGY-ACKER-EXECUTORS 0}]
-    (letlocals
-     (bind sup1 (add-supervisor cluster :id "sup1" :ports [1 2 3 4]))
-     (bind topology1 (thrift/mk-topology
-                      {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 2)}
-                      {}))
-     (bind topology2 (thrift/mk-topology
-                      {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 2)}
-                      {}))
-     (bind state (:storm-cluster-state cluster))
-     (bind changed (capture-changed-workers
-                    (submit-mocked-assignment
-                     (:nimbus cluster)
-                     (:storm-cluster-state cluster)
-                     "topology1"
-                     {TOPOLOGY-WORKERS 2}
-                     topology1
-                     {1 "1"
-                      2 "1"}
-                     {[1 1] ["sup1" 1]
-                      [2 2] ["sup1" 2]}
-                     {["sup1" 1] [0.0 0.0 0.0]
-                      ["sup1" 2] [0.0 0.0 0.0]
-                      })
-                    (submit-mocked-assignment
-                     (:nimbus cluster)
-                     (:storm-cluster-state cluster)
-                     "topology2"
-                     {TOPOLOGY-WORKERS 2}
-                     topology2
-                     {1 "1"
-                      2 "1"}
-                     {[1 1] ["sup1" 1]
-                      [2 2] ["sup1" 2]}
-                     {["sup1" 1] [0.0 0.0 0.0]
-                      ["sup1" 2] [0.0 0.0 0.0]
-                      })
-                    ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
-                    (.rebalance (:nimbus cluster) "topology1" (doto (RebalanceOptions.) (.set_wait_secs 0)))
-                    ))
-     (is (empty? (:launched changed)))
-     (bind options (RebalanceOptions.))
-     (.set_wait_secs options 0)
-     (bind changed (capture-changed-workers
-                    (.rebalance (:nimbus cluster) "topology2" options)
-                    (advance-cluster-time cluster 10)
-                    (heartbeat-workers cluster "sup1" [1 2 3 4])
-                    (advance-cluster-time cluster 10)
-                    ))
-     (validate-launched-once (:launched changed)
-                             {"sup1" [1 2]}
-                             (get-storm-id (:storm-cluster-state cluster) "topology1"))
-     (validate-launched-once (:launched changed)
-                             {"sup1" [3 4]}
-                             (get-storm-id (:storm-cluster-state cluster) "topology2"))
-     )))

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
new file mode 100644
index 0000000..edf534a
--- /dev/null
+++ b/storm-core/test/jvm/org/apache/storm/daemon/supervisor/BasicContainerTest.java
@@ -0,0 +1,483 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.daemon.supervisor;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.storm.Config;
+import org.apache.storm.daemon.supervisor.Container.ContainerType;
+import org.apache.storm.generated.LocalAssignment;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.generated.ProfileRequest;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.utils.LocalState;
+import org.apache.storm.utils.Utils;
+import org.junit.Test;
+
+public class BasicContainerTest {
+    public static class CommandRun {
+        final List<String> cmd;
+        final Map<String, String> env;
+        final File pwd;
+        
+        public CommandRun(List<String> cmd, Map<String, String> env, File pwd) {
+            this.cmd = cmd;
+            this.env = env;
+            this.pwd = pwd;
+        }
+    }
+    
+    public static class MockBasicContainer extends BasicContainer {
+        public MockBasicContainer(ContainerType type, Map<String, Object> conf, String supervisorId, int port,
+                LocalAssignment assignment, LocalState localState,
+                String workerId, Map<String, Object> topoConf, AdvancedFSOps ops, String profileCmd)
+                throws IOException {
+            super(type, conf, supervisorId, port, assignment, localState, workerId, topoConf, ops,
+                    profileCmd);
+        }
+
+        public final List<CommandRun> profileCmds = new ArrayList<>();
+        public final List<CommandRun> workerCmds = new ArrayList<>();
+        
+
+        @Override
+        protected Map<String, Object> readTopoConf() throws IOException {
+            return new HashMap<>();
+        }
+        
+        @Override
+        public void createNewWorkerId() {
+            super.createNewWorkerId();
+        }
+        
+        @Override
+        public List<String> substituteChildopts(Object value, int memOnheap) {
+            return super.substituteChildopts(value, memOnheap);
+        }
+               
+        @Override
+        protected boolean runProfilingCommand(List<String> command, Map<String, String> env, String logPrefix,
+                File targetDir) throws IOException, InterruptedException {
+            profileCmds.add(new CommandRun(command, env, targetDir));
+            return true;
+        }
+        
+        @Override
+        protected void launchWorkerProcess(List<String> command, Map<String, String> env, String logPrefix,
+                ExitCodeCallback processExitCallback, File targetDir) throws IOException {
+            workerCmds.add(new CommandRun(command, env, targetDir));
+        }
+        
+        @Override
+        protected String javaCmd(String cmd) {
+            //avoid system dependent things
+            return cmd;
+        }
+        
+        @Override
+        protected List<String> frameworkClasspath() {
+            //We are not really running anything so make this
+            // simple to check for
+            return Arrays.asList("FRAMEWORK_CP");
+        }
+        
+        @Override
+        protected String javaLibraryPath(String stormRoot, Map<String, Object> conf) {
+            return "JLP";
+        }
+    }
+    
+    @Test
+    public void testCreateNewWorkerId() throws Exception {
+        final String topoId = "test_topology";
+        final int port = 8080;
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        
+        Map<String, Object> superConf = new HashMap<>();
+        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+        
+        LocalState ls = mock(LocalState.class);
+        
+        MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, 
+                "SUPERVISOR", port, la, ls, null, new HashMap<>(), ops, "profile");
+        //null worker id means generate one...
+        
+        assertNotNull(mc._workerId);
+        verify(ls).getApprovedWorkers();
+        Map<String, Integer> expectedNewState = new HashMap<String, Integer>();
+        expectedNewState.put(mc._workerId, port);
+        verify(ls).setApprovedWorkers(expectedNewState);
+    }
+    
+    @Test
+    public void testRecovery() throws Exception {
+        final String topoId = "test_topology";
+        final String workerId = "myWorker";
+        final int port = 8080;
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        
+        Map<String, Integer> workerState = new HashMap<String, Integer>();
+        workerState.put(workerId, port);
+        
+        LocalState ls = mock(LocalState.class);
+        when(ls.getApprovedWorkers()).thenReturn(workerState);
+        
+        Map<String, Object> superConf = new HashMap<>();
+        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+        
+        MockBasicContainer mc = new MockBasicContainer(ContainerType.RECOVER_FULL, superConf, 
+                "SUPERVISOR", port, la, ls, null, new HashMap<>(), ops, "profile");
+        
+        assertEquals(workerId, mc._workerId);
+    }
+    
+    @Test
+    public void testRecoveryMiss() throws Exception {
+        final String topoId = "test_topology";
+        final int port = 8080;
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        
+        Map<String, Integer> workerState = new HashMap<String, Integer>();
+        workerState.put("somethingelse", port+1);
+        
+        LocalState ls = mock(LocalState.class);
+        when(ls.getApprovedWorkers()).thenReturn(workerState);
+        
+        try {
+            new MockBasicContainer(ContainerType.RECOVER_FULL, new HashMap<String, Object>(), 
+                    "SUPERVISOR", port, la, ls, null, new HashMap<>(), null, "profile");
+            fail("Container recovered worker incorrectly");
+        } catch (ContainerRecoveryException e) {
+            //Expected
+        }
+    }
+    
+    @Test
+    public void testCleanUp() throws Exception {
+        final String topoId = "test_topology";
+        final int port = 8080;
+        final String workerId = "worker-id";
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        
+        Map<String, Object> superConf = new HashMap<>();
+        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+        
+        Map<String, Integer> workerState = new HashMap<String, Integer>();
+        workerState.put(workerId, port);
+        
+        LocalState ls = mock(LocalState.class);
+        when(ls.getApprovedWorkers()).thenReturn(new HashMap<>(workerState));
+        
+        MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, 
+                "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+        
+        mc.cleanUp();
+        
+        assertNull(mc._workerId);
+        verify(ls).getApprovedWorkers();
+        Map<String, Integer> expectedNewState = new HashMap<String, Integer>();
+        verify(ls).setApprovedWorkers(expectedNewState);
+    }
+    
+    @Test
+    public void testRunProfiling() throws Exception {
+        final long pid = 100;
+        final String topoId = "test_topology";
+        final int port = 8080;
+        final String workerId = "worker-id";
+        final String stormLocal = ContainerTest.asAbsPath("tmp", "testing");
+        final String topoRoot = ContainerTest.asAbsPath(stormLocal, topoId, String.valueOf(port));
+        final File workerArtifactsPid = ContainerTest.asAbsFile(topoRoot, "worker.pid");
+        
+        final Map<String, Object> superConf = new HashMap<>();
+        superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
+        superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
+        
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        
+        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+        when(ops.slurpString(workerArtifactsPid)).thenReturn(String.valueOf(pid));
+        
+        LocalState ls = mock(LocalState.class);
+        
+        MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, 
+                "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+        
+        //HEAP DUMP
+        ProfileRequest req = new ProfileRequest();
+        req.set_action(ProfileAction.JMAP_DUMP);
+        
+        mc.runProfiling(req, false);
+        
+        assertEquals(1, mc.profileCmds.size());
+        CommandRun cmd = mc.profileCmds.get(0);
+        mc.profileCmds.clear();
+        assertEquals(Arrays.asList("profile", String.valueOf(pid), "jmap", topoRoot), cmd.cmd);
+        assertEquals(new File(topoRoot), cmd.pwd);
+        
+        //JSTACK DUMP
+        req.set_action(ProfileAction.JSTACK_DUMP);
+        
+        mc.runProfiling(req, false);
+        
+        assertEquals(1, mc.profileCmds.size());
+        cmd = mc.profileCmds.get(0);
+        mc.profileCmds.clear();
+        assertEquals(Arrays.asList("profile", String.valueOf(pid), "jstack", topoRoot), cmd.cmd);
+        assertEquals(new File(topoRoot), cmd.pwd);
+        
+        //RESTART
+        req.set_action(ProfileAction.JVM_RESTART);
+        
+        mc.runProfiling(req, false);
+        
+        assertEquals(1, mc.profileCmds.size());
+        cmd = mc.profileCmds.get(0);
+        mc.profileCmds.clear();
+        assertEquals(Arrays.asList("profile", String.valueOf(pid), "kill"), cmd.cmd);
+        assertEquals(new File(topoRoot), cmd.pwd);
+        
+        //JPROFILE DUMP
+        req.set_action(ProfileAction.JPROFILE_DUMP);
+        
+        mc.runProfiling(req, false);
+        
+        assertEquals(1, mc.profileCmds.size());
+        cmd = mc.profileCmds.get(0);
+        mc.profileCmds.clear();
+        assertEquals(Arrays.asList("profile", String.valueOf(pid), "dump", topoRoot), cmd.cmd);
+        assertEquals(new File(topoRoot), cmd.pwd);
+        
+        //JPROFILE START
+        req.set_action(ProfileAction.JPROFILE_STOP);
+        
+        mc.runProfiling(req, false);
+        
+        assertEquals(1, mc.profileCmds.size());
+        cmd = mc.profileCmds.get(0);
+        mc.profileCmds.clear();
+        assertEquals(Arrays.asList("profile", String.valueOf(pid), "start"), cmd.cmd);
+        assertEquals(new File(topoRoot), cmd.pwd);
+        
+        //JPROFILE STOP
+        req.set_action(ProfileAction.JPROFILE_STOP);
+        
+        mc.runProfiling(req, true);
+        
+        assertEquals(1, mc.profileCmds.size());
+        cmd = mc.profileCmds.get(0);
+        mc.profileCmds.clear();
+        assertEquals(Arrays.asList("profile", String.valueOf(pid), "stop", topoRoot), cmd.cmd);
+        assertEquals(new File(topoRoot), cmd.pwd);
+    }
+    
+    private static void setSystemProp(String key, String value) {
+        if (value == null) {
+            System.clearProperty(key);
+        } else {
+            System.setProperty(key, value);
+        }
+    }
+    
+    private static interface Run {
+        public void run() throws Exception;
+    }
+    
+    private static void checkpoint(Run r, String ... newValues) throws Exception {
+        if (newValues.length % 2 != 0) {
+            throw new IllegalArgumentException("Parameters are of the form system property name, new value");
+        }
+        Map<String, String> orig = new HashMap<>();
+        try {
+            for (int index = 0; index < newValues.length; index += 2) {
+                String key = newValues[index];
+                String value = newValues[index + 1];
+                orig.put(key, System.getProperty(key));
+                setSystemProp(key, value);
+            }
+            r.run();
+        } finally {
+            for (Map.Entry<String, String> entry: orig.entrySet()) {
+                setSystemProp(entry.getKey(), entry.getValue());
+            }
+        }
+    }
+    
+    private static <T> void assertListEquals(List<T> a, List<T> b) {
+        if (a == null) {
+            assertNull(b);
+        }
+        if (b == null) {
+            assertNull(a);
+        }
+        int commonLen = Math.min(a.size(), b.size());
+        for (int i = 0; i < commonLen; i++) {
+            assertEquals("at index "+i+"\n"+a+" !=\n"+b+"\n", a.get(i), b.get(i));
+        }
+        
+        assertEquals("size of lists don't match \n"+a+" !=\n"+b, a.size(), b.size());
+    }
+    
+    @Test
+    public void testLaunch() throws Exception {
+        final String topoId = "test_topology";
+        final int port = 8080;
+        final String stormHome = ContainerTest.asAbsPath("tmp", "storm-home");
+        final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath();
+        final String workerId = "worker-id";
+        final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local");
+        final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", topoId);
+        final File stormcode = new File(distRoot, "stormcode.ser");
+        final File stormjar = new File(distRoot, "stormjar.jar");
+        final String log4jdir = ContainerTest.asAbsPath(stormHome, "conf");
+        final String workerConf = ContainerTest.asAbsPath(log4jdir, "worker.xml");
+        final String workerRoot = ContainerTest.asAbsPath(stormLocal, "workers", workerId);
+        final String workerTmpDir = ContainerTest.asAbsPath(workerRoot, "tmp");
+        
+        final StormTopology st = new StormTopology();
+        st.set_spouts(new HashMap<>());
+        st.set_bolts(new HashMap<>());
+        st.set_state_spouts(new HashMap<>());
+        byte [] serializedState = Utils.gzip(Utils.thriftSerialize(st));
+        
+        final Map<String, Object> superConf = new HashMap<>();
+        superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
+        superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
+        superConf.put(Config.STORM_LOG4J2_CONF_DIR, log4jdir);
+        
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        
+        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+        when(ops.slurp(stormcode)).thenReturn(serializedState);
+        
+        LocalState ls = mock(LocalState.class);
+        
+        
+        checkpoint(() -> {
+            MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, 
+                    "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+
+            mc.launch();
+
+            assertEquals(1, mc.workerCmds.size());
+            CommandRun cmd = mc.workerCmds.get(0);
+            mc.workerCmds.clear();
+            assertListEquals(Arrays.asList(
+                    "java",
+                    "-cp",
+                    "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
+                    "-Dlogging.sensitivity=S3",
+                    "-Dlogfile.name=worker.log",
+                    "-Dstorm.home=" + stormHome,
+                    "-Dworkers.artifacts=" + stormLocal,
+                    "-Dstorm.id=" + topoId,
+                    "-Dworker.id=" + workerId,
+                    "-Dworker.port=" + port,
+                    "-Dstorm.log.dir=" + stormLogDir,
+                    "-Dlog4j.configurationFile=" + workerConf,
+                    "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
+                    "-Dstorm.local.dir=" + stormLocal,
+                    "org.apache.storm.LogWriter",
+                    "java",
+                    "-server",
+                    "-Dlogging.sensitivity=S3",
+                    "-Dlogfile.name=worker.log",
+                    "-Dstorm.home=" + stormHome,
+                    "-Dworkers.artifacts=" + stormLocal,
+                    "-Dstorm.id=" + topoId,
+                    "-Dworker.id=" + workerId,
+                    "-Dworker.port=" + port,
+                    "-Dstorm.log.dir=" + stormLogDir,
+                    "-Dlog4j.configurationFile=" + workerConf,
+                    "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector",
+                    "-Dstorm.local.dir=" + stormLocal,
+                    "-Djava.library.path=JLP",
+                    "-Dstorm.conf.file=",
+                    "-Dstorm.options=",
+                    "-Djava.io.tmpdir="+workerTmpDir,
+                    "-cp",
+                    "FRAMEWORK_CP:" + stormjar.getAbsolutePath(),
+                    "org.apache.storm.daemon.worker", 
+                    topoId, 
+                    "SUPERVISOR",
+                    String.valueOf(port),
+                    workerId
+                    ), cmd.cmd);
+            assertEquals(new File(workerRoot), cmd.pwd);
+          }, 
+          "storm.home", stormHome,
+          "storm.log.dir", stormLogDir);
+    }
+    
+    @Test
+    public void testSubstChildOpts() throws Exception {
+        String workerId = "w-01";
+        String topoId = "s-01";
+        int port = 9999;
+        int memOnheap = 512;
+        
+        LocalAssignment la = new LocalAssignment();
+        la.set_topology_id(topoId);
+        
+        Map<String, Object> superConf = new HashMap<>();
+        
+        AdvancedFSOps ops = mock(AdvancedFSOps.class);
+        when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
+        
+        LocalState ls = mock(LocalState.class);
+        
+        MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, 
+                "SUPERVISOR", port, la, ls, workerId, new HashMap<>(), ops, "profile");
+        
+        assertListEquals(Arrays.asList(
+                "-Xloggc:/tmp/storm/logs/gc.worker-9999-s-01-w-01-9999.log",
+                "-Xms256m",
+                "-Xmx512m"),
+                mc.substituteChildopts("-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log -Xms256m -Xmx%HEAP-MEM%m", memOnheap));
+        
+        assertListEquals(Arrays.asList(
+                "-Xloggc:/tmp/storm/logs/gc.worker-9999-s-01-w-01-9999.log",
+                "-Xms256m",
+                "-Xmx512m"),
+                mc.substituteChildopts(Arrays.asList("-Xloggc:/tmp/storm/logs/gc.worker-%ID%-%TOPOLOGY-ID%-%WORKER-ID%-%WORKER-PORT%.log","-Xms256m","-Xmx%HEAP-MEM%m"), memOnheap));
+        
+        assertListEquals(Collections.emptyList(), 
+                mc.substituteChildopts(null));
+    }
+}


[11/21] storm git commit: Merge branch 'STORM-2117' of https://github.com/HeartSaVioR/storm into STORM-2117

Posted by ka...@apache.org.
Merge branch 'STORM-2117' of https://github.com/HeartSaVioR/storm into STORM-2117

STROM-2117:  Supervisor V2 with local mode extracts resources directory to the wrong directory


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fb42b960
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fb42b960
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fb42b960

Branch: refs/heads/1.x-branch
Commit: fb42b960fae0a06f13f2ee83ddf2344521c1c8e8
Parents: 24ff529
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Sep 23 12:28:03 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Sep 23 12:58:06 2016 -0500

----------------------------------------------------------------------
 .../src/jvm/org/apache/storm/localizer/AsyncLocalizer.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/fb42b960/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java b/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
index 7887281..d3e3925 100644
--- a/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
+++ b/storm-core/src/jvm/org/apache/storm/localizer/AsyncLocalizer.java
@@ -192,12 +192,12 @@ public class AsyncLocalizer implements ILocalizer, Shutdownable {
 
             if (resourcesJar != null) {
                 LOG.info("Extracting resources from jar at {} to {}", resourcesJar, targetDir);
-                Utils.extractDirFromJar(resourcesJar, ConfigUtils.RESOURCES_SUBDIR, _stormRoot);
+                Utils.extractDirFromJar(resourcesJar, ConfigUtils.RESOURCES_SUBDIR, new File(targetDir));
             } else if (url != null) {
                 LOG.info("Copying resources at {} to {} ", url.toString(), targetDir);
                 if ("jar".equals(url.getProtocol())) {
                     JarURLConnection urlConnection = (JarURLConnection) url.openConnection();
-                    Utils.extractDirFromJar(urlConnection.getJarFileURL().getFile(), ConfigUtils.RESOURCES_SUBDIR, _stormRoot);
+                    Utils.extractDirFromJar(urlConnection.getJarFileURL().getFile(), ConfigUtils.RESOURCES_SUBDIR, new File(targetDir));
                 } else {
                     _fsOps.copyDirectory(new File(url.getFile()), new File(targetDir));
                 }


[08/21] storm git commit: STORM-2018: Supervisor V2

Posted by ka...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/clj/org/apache/storm/testing.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/testing.clj b/storm-core/src/clj/org/apache/storm/testing.clj
index 565d1b9..4029be1 100644
--- a/storm-core/src/clj/org/apache/storm/testing.clj
+++ b/storm-core/src/clj/org/apache/storm/testing.clj
@@ -17,12 +17,13 @@
 (ns org.apache.storm.testing
   (:require [org.apache.storm.daemon
              [nimbus :as nimbus]
-             [supervisor :as supervisor]
+             [local-supervisor :as local-supervisor]
              [common :as common]
              [worker :as worker]
              [executor :as executor]])
-  (:require [org.apache.storm [process-simulator :as psim]])
-  (:import [org.apache.commons.io FileUtils])
+  (:import [org.apache.commons.io FileUtils]
+           [org.apache.storm ProcessSimulator]
+           [org.apache.storm.daemon.supervisor Supervisor StandaloneSupervisor SupervisorUtils])
   (:import [java.io File])
   (:import [java.util HashMap ArrayList])
   (:import [java.util.concurrent.atomic AtomicInteger])
@@ -31,6 +32,7 @@
   (:import [org.apache.storm.tuple Fields Tuple TupleImpl])
   (:import [org.apache.storm.task TopologyContext])
   (:import [org.apache.storm.generated GlobalStreamId Bolt KillOptions])
+  (:import [org.apache.storm.messaging IContext])
   (:import [org.apache.storm.testing FeederSpout FixedTupleSpout FixedTuple
             TupleCaptureBolt SpoutTracker BoltTracker NonRichBoltTracker
             TestWordSpout MemoryTransactionalSpout])
@@ -108,8 +110,10 @@
                                conf
                                {STORM-LOCAL-DIR tmp-dir
                                 SUPERVISOR-SLOTS-PORTS port-ids})
-        id-fn (if id (fn [] id) supervisor/generate-supervisor-id)
-        daemon (with-var-roots [supervisor/generate-supervisor-id id-fn] (supervisor/mk-supervisor supervisor-conf (:shared-context cluster-map) (supervisor/standalone-supervisor)))]
+        id (or id (Utils/uuid))
+        isupervisor (proxy [StandaloneSupervisor] []
+                        (generateSupervisorId [] id))
+        daemon (local-supervisor/mk-local-supervisor supervisor-conf (:shared-context cluster-map) isupervisor)]
     (swap! (:supervisors cluster-map) conj daemon)
     (swap! (:tmp-dirs cluster-map) conj tmp-dir)
     daemon))
@@ -135,18 +139,19 @@
   (let [zk-tmp (local-temp-path)
         [zk-port zk-handle] (if-not (contains? daemon-conf STORM-ZOOKEEPER-SERVERS)
                               (zk/mk-inprocess-zookeeper zk-tmp))
+        nimbus-tmp (local-temp-path)
         daemon-conf (merge (read-storm-config)
                            {TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true
                             ZMQ-LINGER-MILLIS 0
                             TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS false
                             TOPOLOGY-TRIDENT-BATCH-EMIT-INTERVAL-MILLIS 50
                             STORM-CLUSTER-MODE "local"
-                            BLOBSTORE-SUPERUSER (System/getProperty "user.name")}
+                            BLOBSTORE-SUPERUSER (System/getProperty "user.name")
+                            BLOBSTORE-DIR nimbus-tmp}
                            (if-not (contains? daemon-conf STORM-ZOOKEEPER-SERVERS)
                              {STORM-ZOOKEEPER-PORT zk-port
                               STORM-ZOOKEEPER-SERVERS ["localhost"]})
                            daemon-conf)
-        nimbus-tmp (local-temp-path)
         port-counter (mk-counter supervisor-slot-port-min)
         nimbus (nimbus/service-handler
                 (assoc daemon-conf STORM-LOCAL-DIR nimbus-tmp)
@@ -172,17 +177,17 @@
     cluster-map))
 
 (defn get-supervisor [cluster-map supervisor-id]
-  (let [finder-fn #(= (.get-id %) supervisor-id)]
+  (let [finder-fn #(= (.getId %) supervisor-id)]
     (find-first finder-fn @(:supervisors cluster-map))))
 
 (defn kill-supervisor [cluster-map supervisor-id]
-  (let [finder-fn #(= (.get-id %) supervisor-id)
+  (let [finder-fn #(= (.getId %) supervisor-id)
         supervisors @(:supervisors cluster-map)
         sup (find-first finder-fn
                         supervisors)]
     ;; tmp-dir will be taken care of by shutdown
     (reset! (:supervisors cluster-map) (remove-first finder-fn supervisors))
-    (.shutdown sup)))
+    (.close sup)))
 
 (defn kill-local-storm-cluster [cluster-map]
   (.shutdown (:nimbus cluster-map))
@@ -196,10 +201,10 @@
   (.close (:state cluster-map))
   (.disconnect (:storm-cluster-state cluster-map))
   (doseq [s @(:supervisors cluster-map)]
-    (.shutdown-all-workers s)
+    (.shutdownAllWorkers s)
     ;; race condition here? will it launch the workers again?
-    (supervisor/kill-supervisor s))
-  (psim/kill-all-processes)
+    (.close s))
+  (ProcessSimulator/killAllProcesses)
   (if (not-nil? (:zookeeper cluster-map))
     (do
       (log-message "Shutting down in process zookeeper")
@@ -235,19 +240,23 @@
     (while-timeout timeout-ms (not (apredicate))
       (Time/sleep 100))))
 
+(defn is-supervisor-waiting [^Supervisor supervisor]
+   (.isWaiting supervisor))
+
+
 (defn wait-until-cluster-waiting
   "Wait until the cluster is idle. Should be used with time simulation."
   ([cluster-map] (wait-until-cluster-waiting cluster-map TEST-TIMEOUT-MS))
   ([cluster-map timeout-ms]
   ;; wait until all workers, supervisors, and nimbus is waiting
   (let [supervisors @(:supervisors cluster-map)
-        workers (filter (partial satisfies? common/DaemonCommon) (psim/all-processes))
+        workers (filter (partial satisfies? common/DaemonCommon) (clojurify-structure (ProcessSimulator/getAllProcessHandles)))
         daemons (concat
                   [(:nimbus cluster-map)]
-                  supervisors
                   ; because a worker may already be dead
                   workers)]
-    (while-timeout timeout-ms (not (every? (memfn waiting?) daemons))
+    (while-timeout timeout-ms (or (not (every? (memfn waiting?) daemons))
+                                (not (every? is-supervisor-waiting supervisors)))
                    (Thread/sleep (rand-int 20))
                    ;;      (doseq [d daemons]
                    ;;        (if-not ((memfn waiting?) d)
@@ -278,7 +287,8 @@
                f# (future (while @keep-waiting?# (simulate-wait ~cluster-sym)))]
            (kill-local-storm-cluster ~cluster-sym)
            (reset! keep-waiting?# false)
-            @f#)))))
+            @f#
+           (.setSubject (ReqContext/context) nil))))))
 
 (defmacro with-simulated-time-local-cluster
   [& args]
@@ -360,35 +370,6 @@
         worker->port (ls-approved-workers supervisor-state)]
     (worker->port worker-id)))
 
-(defn mk-capture-shutdown-fn
-  [capture-atom]
-  (let [existing-fn supervisor/shutdown-worker]
-    (fn [supervisor worker-id]
-      (let [conf (:conf supervisor)
-            supervisor-id (:supervisor-id supervisor)
-            port (find-worker-port conf worker-id)
-            existing (get @capture-atom [supervisor-id port] 0)]
-        (swap! capture-atom assoc [supervisor-id port] (inc existing))
-        (existing-fn supervisor worker-id)))))
-
-(defmacro capture-changed-workers
-  [& body]
-  `(let [launch-captured# (atom {})
-         shutdown-captured# (atom {})]
-     (with-var-roots [supervisor/launch-worker (mk-capture-launch-fn launch-captured#)
-                      supervisor/shutdown-worker (mk-capture-shutdown-fn shutdown-captured#)]
-                     ~@body
-                     {:launched @launch-captured#
-                      :shutdown @shutdown-captured#})))
-
-(defmacro capture-launched-workers
-  [& body]
-  `(:launched (capture-changed-workers ~@body)))
-
-(defmacro capture-shutdown-workers
-  [& body]
-  `(:shutdown (capture-changed-workers ~@body)))
-
 (defnk aggregated-stat
   [cluster-map storm-name stat-key :component-ids nil]
   (let [state (:storm-cluster-state cluster-map)
@@ -660,6 +641,7 @@
                      ;; (println "Spout emitted: " (global-amt track-id "spout-emitted"))
                      ;; (println "Processed: " (global-amt track-id "processed"))
                      ;; (println "Transferred: " (global-amt track-id "transferred"))
+                    (advance-time-secs! 1)
                     (Thread/sleep (rand-int 200)))
       (reset! (:last-spout-emit tracked-topology) target))))
 

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/ProcessSimulator.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/ProcessSimulator.java b/storm-core/src/jvm/org/apache/storm/ProcessSimulator.java
new file mode 100644
index 0000000..202df12
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/ProcessSimulator.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm;
+import org.apache.storm.daemon.Shutdownable;
+
+import java.util.Collection;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * In local mode, {@code ProcessSimulator} keeps track of Shutdownable objects
+ * in place of actual processes (in cluster mode).
+ */
+public class ProcessSimulator {
+    private static Logger LOG = LoggerFactory.getLogger(ProcessSimulator.class);
+    private static Object lock = new Object();
+    protected static ConcurrentHashMap<String, Shutdownable> processMap = new ConcurrentHashMap<String, Shutdownable>();
+
+    /**
+     * Register a process' handle
+     * 
+     * @param pid
+     * @param shutdownable
+     */
+    public static void registerProcess(String pid, Shutdownable shutdownable) {
+        processMap.put(pid, shutdownable);
+    }
+
+    /**
+     * Get all process handles
+     * 
+     * @return
+     */
+    public static Collection<Shutdownable> getAllProcessHandles() {
+        return processMap.values();
+    }
+
+    /**
+     * Kill a process
+     * 
+     * @param pid
+     */
+    public static void killProcess(String pid) {
+        synchronized (lock) {
+            LOG.info("Begin killing process " + pid);
+            Shutdownable shutdownHandle = processMap.get(pid);
+            if (shutdownHandle != null) {
+                shutdownHandle.shutdown();
+            }
+            processMap.remove(pid);
+            LOG.info("Successfully killed process " + pid);
+        }
+    }
+
+    /**
+     * Kill all processes
+     */
+    public static void killAllProcesses() {
+        Set<String> pids = processMap.keySet();
+        for (String pid : pids) {
+            killProcess(pid);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/StormTimer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/StormTimer.java b/storm-core/src/jvm/org/apache/storm/StormTimer.java
new file mode 100644
index 0000000..4be62b4
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/StormTimer.java
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm;
+
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Comparator;
+import java.util.Random;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * The timer defined in this file is very similar to java.util.Timer, except
+ * it integrates with Storm's time simulation capabilities. This lets us test
+ * code that does asynchronous work on the timer thread
+ */
+
+public class StormTimer implements AutoCloseable {
+    private static final Logger LOG = LoggerFactory.getLogger(StormTimer.class);
+
+    public static class QueueEntry {
+        public final Long endTimeMs;
+        public final Runnable func;
+        public final String id;
+
+        public QueueEntry(Long endTimeMs, Runnable func, String id) {
+            this.endTimeMs = endTimeMs;
+            this.func = func;
+            this.id = id;
+        }
+    }
+
+    public static class StormTimerTask extends Thread {
+
+        //initialCapacity set to 11 since its the default inital capacity of PriorityBlockingQueue
+        private PriorityBlockingQueue<QueueEntry> queue = new PriorityBlockingQueue<QueueEntry>(11, new Comparator<QueueEntry>() {
+            @Override
+            public int compare(QueueEntry o1, QueueEntry o2) {
+                return o1.endTimeMs.intValue() - o2.endTimeMs.intValue();
+            }
+        });
+
+        // boolean to indicate whether timer is active
+        private AtomicBoolean active = new AtomicBoolean(false);
+
+        // function to call when timer is killed
+        private Thread.UncaughtExceptionHandler onKill;
+
+        //random number generator
+        private Random random = new Random();
+
+        @Override
+        public void run() {
+            while (this.active.get()) {
+                QueueEntry queueEntry = null;
+                try {
+                    queueEntry = this.queue.peek();
+                    if ((queueEntry != null) && (Time.currentTimeMillis() >= queueEntry.endTimeMs)) {
+                        // It is imperative to not run the function
+                        // inside the timer lock. Otherwise, it is
+                        // possible to deadlock if the fn deals with
+                        // other locks, like the submit lock.
+                        this.queue.remove(queueEntry);
+                        queueEntry.func.run();
+                    } else if (queueEntry != null) {
+                        //  If any events are scheduled, sleep until
+                        // event generation. If any recurring events
+                        // are scheduled then we will always go
+                        // through this branch, sleeping only the
+                        // exact necessary amount of time. We give
+                        // an upper bound, e.g. 1000 millis, to the
+                        // sleeping time, to limit the response time
+                        // for detecting any new event within 1 secs.
+                        Time.sleep(Math.min(1000, (queueEntry.endTimeMs - Time.currentTimeMillis())));
+                    } else {
+                        // Otherwise poll to see if any new event
+                        // was scheduled. This is, in essence, the
+                        // response time for detecting any new event
+                        // schedulings when there are no scheduled
+                        // events.
+                        Time.sleep(1000);
+                    }
+                } catch (Throwable e) {
+                    if (!(Utils.exceptionCauseIsInstanceOf(InterruptedException.class, e))) {
+                        this.onKill.uncaughtException(this, e);
+                        this.setActive(false);
+                    }
+                }
+            }
+        }
+
+        public void setOnKillFunc(Thread.UncaughtExceptionHandler onKill) {
+            this.onKill = onKill;
+        }
+
+        public void setActive(boolean flag) {
+            this.active.set(flag);
+        }
+
+        public boolean isActive() {
+            return this.active.get();
+        }
+
+        public void add(QueueEntry queueEntry) {
+            this.queue.add(queueEntry);
+        }
+    }
+
+    //task to run
+    private StormTimerTask task = new StormTimerTask();
+
+    /**
+     * Makes a Timer in the form of a StormTimerTask Object
+     * @param name name of the timer
+     * @param onKill function to call when timer is killed unexpectedly
+     * @return StormTimerTask object that was initialized
+     */
+    public StormTimer (String name, Thread.UncaughtExceptionHandler onKill) {
+        if (onKill == null) {
+            throw new RuntimeException("onKill func is null!");
+        }
+        if (name == null) {
+            this.task.setName("timer");
+        } else {
+            this.task.setName(name);
+        }
+        this.task.setOnKillFunc(onKill);
+        this.task.setActive(true);
+
+        this.task.setDaemon(true);
+        this.task.setPriority(Thread.MAX_PRIORITY);
+        this.task.start();
+    }
+
+    /**
+     * Schedule a function to be executed in the timer
+     * @param delaySecs the number of seconds to delay before running the function
+     * @param func the function to run
+     * @param checkActive whether to check is the timer is active
+     * @param jitterMs add jitter to the run
+     */
+    public void schedule(int delaySecs, Runnable func, boolean checkActive, int jitterMs) {
+        if (func == null) {
+            throw new RuntimeException("function to schedule is null!");
+        }
+        if (checkActive) {
+            checkActive();
+        }
+        String id = Utils.uuid();
+        long endTimeMs = Time.currentTimeMillis() + Time.secsToMillisLong(delaySecs);
+        if (jitterMs > 0) {
+            endTimeMs = this.task.random.nextInt(jitterMs) + endTimeMs;
+        }
+        task.add(new QueueEntry(endTimeMs, func, id));
+    }
+
+    public void schedule(int delaySecs, Runnable func) {
+        schedule(delaySecs, func, true, 0);
+    }
+
+    /**
+     * Schedule a function to run recurrently
+     * @param delaySecs the number of seconds to delay before running the function
+     * @param recurSecs the time between each invocation
+     * @param func the function to run
+     */
+    public void scheduleRecurring(int delaySecs, final int recurSecs, final Runnable func) {
+        schedule(delaySecs, new Runnable() {
+            @Override
+            public void run() {
+                func.run();
+                // This avoids a race condition with cancel-timer.
+                schedule(recurSecs, this, false, 0);
+            }
+        });
+    }
+
+    /**
+     * schedule a function to run recurrently with jitter
+     * @param delaySecs the number of seconds to delay before running the function
+     * @param recurSecs the time between each invocation
+     * @param jitterMs jitter added to the run
+     * @param func the function to run
+     */
+    public void scheduleRecurringWithJitter(int delaySecs, final int recurSecs, final int jitterMs, final Runnable func) {
+        schedule(delaySecs, new Runnable() {
+            @Override
+            public void run() {
+                func.run();
+                // This avoids a race condition with cancel-timer.
+                schedule(recurSecs, this, false, jitterMs);
+            }
+        });
+    }
+
+    /**
+     * check if timer is active
+     */
+    private void checkActive() {
+        if (!this.task.isActive()) {
+            throw new IllegalStateException("Timer is not active");
+        }
+    }
+
+    /**
+     * cancel timer
+     */
+
+    @Override
+    public void close() throws Exception {
+        checkActive();
+        this.task.setActive(false);
+        this.task.interrupt();
+        this.task.join();
+    }
+
+    /**
+     * is timer waiting. Used in timer simulation
+     */
+    public boolean isTimerWaiting() {
+        return Time.isThreadWaiting(task);
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java b/storm-core/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java
new file mode 100644
index 0000000..043dd0c
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/callback/DefaultWatcherCallBack.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.callback;
+
+import org.apache.storm.zookeeper.ZkEventTypes;
+import org.apache.storm.zookeeper.ZkKeeperStates;
+import org.apache.zookeeper.Watcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DefaultWatcherCallBack implements WatcherCallBack {
+
+    private static Logger LOG = LoggerFactory.getLogger(DefaultWatcherCallBack.class);
+
+    @Override
+    public void execute(Watcher.Event.KeeperState state, Watcher.Event.EventType type, String path) {
+        LOG.debug("Zookeeper state update:  {}, {}, {}", ZkKeeperStates.getStateName(state), ZkEventTypes.getTypeName(type), path);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/callback/WatcherCallBack.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/callback/WatcherCallBack.java b/storm-core/src/jvm/org/apache/storm/callback/WatcherCallBack.java
new file mode 100644
index 0000000..41a50ec
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/callback/WatcherCallBack.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.callback;
+
+import org.apache.zookeeper.Watcher;
+
+public interface WatcherCallBack {
+    public void execute(Watcher.Event.KeeperState state, Watcher.Event.EventType type, String path);
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java b/storm-core/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java
new file mode 100644
index 0000000..75b0e99
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/callback/ZKStateChangedCallback.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.callback;
+
+import org.apache.zookeeper.Watcher;
+
+public interface ZKStateChangedCallback {
+    public void changed(Watcher.Event.EventType type, String path);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/ClusterStateFactory.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/ClusterStateFactory.java b/storm-core/src/jvm/org/apache/storm/cluster/ClusterStateFactory.java
index 6474d82..fcf5db0 100644
--- a/storm-core/src/jvm/org/apache/storm/cluster/ClusterStateFactory.java
+++ b/storm-core/src/jvm/org/apache/storm/cluster/ClusterStateFactory.java
@@ -21,7 +21,9 @@ import clojure.lang.APersistentMap;
 import java.util.List;
 import org.apache.zookeeper.data.ACL;
 
-public interface ClusterStateFactory {
+import org.apache.storm.cluster.StateStorageFactory;
+
+public interface ClusterStateFactory extends StateStorageFactory {
     
     ClusterState mkState(APersistentMap config, APersistentMap auth_conf, List<ACL> acls, ClusterStateContext context);
 

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/ClusterUtils.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/ClusterUtils.java b/storm-core/src/jvm/org/apache/storm/cluster/ClusterUtils.java
new file mode 100644
index 0000000..96c177b
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/ClusterUtils.java
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import org.apache.storm.Config;
+import org.apache.storm.generated.ClusterWorkerHeartbeat;
+import org.apache.storm.generated.ExecutorInfo;
+import org.apache.storm.generated.ExecutorStats;
+import org.apache.storm.generated.ProfileAction;
+import org.apache.storm.utils.Utils;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ClusterUtils {
+
+    public static final String ZK_SEPERATOR = "/";
+
+    public static final String ASSIGNMENTS_ROOT = "assignments";
+    public static final String CODE_ROOT = "code";
+    public static final String STORMS_ROOT = "storms";
+    public static final String SUPERVISORS_ROOT = "supervisors";
+    public static final String WORKERBEATS_ROOT = "workerbeats";
+    public static final String BACKPRESSURE_ROOT = "backpressure";
+    public static final String ERRORS_ROOT = "errors";
+    public static final String BLOBSTORE_ROOT = "blobstore";
+    public static final String BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_ROOT = "blobstoremaxkeysequencenumber";
+    public static final String NIMBUSES_ROOT = "nimbuses";
+    public static final String CREDENTIALS_ROOT = "credentials";
+    public static final String LOGCONFIG_ROOT = "logconfigs";
+    public static final String PROFILERCONFIG_ROOT = "profilerconfigs";
+
+    public static final String ASSIGNMENTS_SUBTREE = ZK_SEPERATOR + ASSIGNMENTS_ROOT;
+    public static final String STORMS_SUBTREE = ZK_SEPERATOR + STORMS_ROOT;
+    public static final String SUPERVISORS_SUBTREE = ZK_SEPERATOR + SUPERVISORS_ROOT;
+    public static final String WORKERBEATS_SUBTREE = ZK_SEPERATOR + WORKERBEATS_ROOT;
+    public static final String BACKPRESSURE_SUBTREE = ZK_SEPERATOR + BACKPRESSURE_ROOT;
+    public static final String ERRORS_SUBTREE = ZK_SEPERATOR + ERRORS_ROOT;
+    public static final String BLOBSTORE_SUBTREE = ZK_SEPERATOR + BLOBSTORE_ROOT;
+    public static final String BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE = ZK_SEPERATOR + BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_ROOT;
+    public static final String NIMBUSES_SUBTREE = ZK_SEPERATOR + NIMBUSES_ROOT;
+    public static final String CREDENTIALS_SUBTREE = ZK_SEPERATOR + CREDENTIALS_ROOT;
+    public static final String LOGCONFIG_SUBTREE = ZK_SEPERATOR + LOGCONFIG_ROOT;
+    public static final String PROFILERCONFIG_SUBTREE = ZK_SEPERATOR + PROFILERCONFIG_ROOT;
+
+    // A singleton instance allows us to mock delegated static methods in our
+    // tests by subclassing.
+    private static final ClusterUtils INSTANCE = new ClusterUtils();
+    private static ClusterUtils _instance = INSTANCE;
+
+    /**
+     * Provide an instance of this class for delegates to use. To mock out delegated methods, provide an instance of a subclass that overrides the
+     * implementation of the delegated method.
+     *
+     * @param u a Cluster instance
+     */
+    public static void setInstance(ClusterUtils u) {
+        _instance = u;
+    }
+
+    /**
+     * Resets the singleton instance to the default. This is helpful to reset the class to its original functionality when mocking is no longer desired.
+     */
+    public static void resetInstance() {
+        _instance = INSTANCE;
+    }
+
+    public static List<ACL> mkTopoOnlyAcls(Map topoConf) throws NoSuchAlgorithmException {
+        List<ACL> aclList = null;
+        String payload = (String) topoConf.get(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD);
+        if (Utils.isZkAuthenticationConfiguredTopology(topoConf)) {
+            aclList = new ArrayList<>();
+            ACL acl1 = ZooDefs.Ids.CREATOR_ALL_ACL.get(0);
+            aclList.add(acl1);
+            ACL acl2 = new ACL(ZooDefs.Perms.READ, new Id("digest", DigestAuthenticationProvider.generateDigest(payload)));
+            aclList.add(acl2);
+        }
+        return aclList;
+    }
+
+    public static String supervisorPath(String id) {
+        return SUPERVISORS_SUBTREE + ZK_SEPERATOR + id;
+    }
+
+    public static String assignmentPath(String id) {
+        return ASSIGNMENTS_SUBTREE + ZK_SEPERATOR + id;
+    }
+
+    public static String blobstorePath(String key) {
+        return BLOBSTORE_SUBTREE + ZK_SEPERATOR + key;
+    }
+
+    public static String blobstoreMaxKeySequenceNumberPath(String key) {
+        return BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE + ZK_SEPERATOR + key;
+    }
+
+    public static String nimbusPath(String id) {
+        return NIMBUSES_SUBTREE + ZK_SEPERATOR + id;
+    }
+
+    public static String stormPath(String id) {
+        return STORMS_SUBTREE + ZK_SEPERATOR + id;
+    }
+
+    public static String workerbeatStormRoot(String stormId) {
+        return WORKERBEATS_SUBTREE + ZK_SEPERATOR + stormId;
+    }
+
+    public static String workerbeatPath(String stormId, String node, Long port) {
+        return workerbeatStormRoot(stormId) + ZK_SEPERATOR + node + "-" + port;
+    }
+
+    public static String backpressureStormRoot(String stormId) {
+        return BACKPRESSURE_SUBTREE + ZK_SEPERATOR + stormId;
+    }
+
+    public static String backpressurePath(String stormId, String node, Long port) {
+        return backpressureStormRoot(stormId) + ZK_SEPERATOR + node + "-" + port;
+    }
+
+    public static String errorStormRoot(String stormId) {
+        return ERRORS_SUBTREE + ZK_SEPERATOR + stormId;
+    }
+
+    public static String errorPath(String stormId, String componentId) {
+        try {
+            return errorStormRoot(stormId) + ZK_SEPERATOR + URLEncoder.encode(componentId, "UTF-8");
+        } catch (UnsupportedEncodingException e) {
+            throw Utils.wrapInRuntime(e);
+        }
+    }
+
+    public static String lastErrorPath(String stormId, String componentId) {
+        return errorPath(stormId, componentId) + "-last-error";
+    }
+
+    public static String credentialsPath(String stormId) {
+        return CREDENTIALS_SUBTREE + ZK_SEPERATOR + stormId;
+    }
+
+    public static String logConfigPath(String stormId) {
+        return LOGCONFIG_SUBTREE + ZK_SEPERATOR + stormId;
+    }
+
+    public static String profilerConfigPath(String stormId) {
+        return PROFILERCONFIG_SUBTREE + ZK_SEPERATOR + stormId;
+    }
+
+    public static String profilerConfigPath(String stormId, String host, Long port, ProfileAction requestType) {
+        return profilerConfigPath(stormId) + ZK_SEPERATOR + host + "_" + port + "_" + requestType;
+    }
+
+    public static <T> T maybeDeserialize(byte[] serialized, Class<T> clazz) {
+        if (serialized != null) {
+            return Utils.deserialize(serialized, clazz);
+        }
+        return null;
+    }
+
+    /**
+     * Ensures that we only return heartbeats for executors assigned to this worker
+     * @param executors
+     * @param workerHeartbeat
+     * @return
+     */
+    public static Map<ExecutorInfo, ExecutorBeat> convertExecutorBeats(List<ExecutorInfo> executors, ClusterWorkerHeartbeat workerHeartbeat) {
+        Map<ExecutorInfo, ExecutorBeat> executorWhb = new HashMap<>();
+        Map<ExecutorInfo, ExecutorStats> executorStatsMap = workerHeartbeat.get_executor_stats();
+        for (ExecutorInfo executor : executors) {
+            if (executorStatsMap.containsKey(executor)) {
+                int time = workerHeartbeat.get_time_secs();
+                int uptime = workerHeartbeat.get_uptime_secs();
+                ExecutorStats executorStats = workerHeartbeat.get_executor_stats().get(executor);
+                ExecutorBeat executorBeat = new ExecutorBeat(time, uptime, executorStats);
+                executorWhb.put(executor, executorBeat);
+            }
+        }
+        return executorWhb;
+    }
+
+    public IStormClusterState mkStormClusterStateImpl(Object stateStorage, List<ACL> acls, ClusterStateContext context) throws Exception {
+        if (stateStorage instanceof IStateStorage) {
+            return new StormClusterStateImpl((IStateStorage) stateStorage, acls, context, false);
+        } else {
+            IStateStorage Storage = _instance.mkStateStorageImpl((Map) stateStorage, (Map) stateStorage, acls, context);
+            return new StormClusterStateImpl(Storage, acls, context, true);
+        }
+    }
+
+    public IStateStorage mkStateStorageImpl(Map config, Map auth_conf, List<ACL> acls, ClusterStateContext context) throws Exception {
+        String className = null;
+        IStateStorage stateStorage = null;
+        if (config.get(Config.STORM_CLUSTER_STATE_STORE) != null) {
+            className = (String) config.get(Config.STORM_CLUSTER_STATE_STORE);
+        } else {
+            className = "org.apache.storm.cluster.ZKStateStorageFactory";
+        }
+        Class clazz = Class.forName(className);
+        StateStorageFactory storageFactory = (StateStorageFactory) clazz.newInstance();
+        stateStorage = storageFactory.mkStore(config, auth_conf, acls, context);
+        return stateStorage;
+    }
+
+    public static IStateStorage mkStateStorage(Map config, Map auth_conf, List<ACL> acls, ClusterStateContext context) throws Exception {
+        return _instance.mkStateStorageImpl(config, auth_conf, acls, context);
+    }
+
+    public static IStormClusterState mkStormClusterState(Object StateStorage, List<ACL> acls, ClusterStateContext context) throws Exception {
+        return _instance.mkStormClusterStateImpl(StateStorage, acls, context);
+    }
+
+    public static String stringifyError(Throwable error) {
+        StringWriter result = new StringWriter();
+        PrintWriter printWriter = new PrintWriter(result);
+        error.printStackTrace(printWriter);
+        return result.toString();
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/ExecutorBeat.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/ExecutorBeat.java b/storm-core/src/jvm/org/apache/storm/cluster/ExecutorBeat.java
new file mode 100644
index 0000000..b32615e
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/ExecutorBeat.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import org.apache.storm.generated.ExecutorStats;
+
+public class ExecutorBeat {
+    private final int timeSecs;
+    private final int uptime;
+    private final ExecutorStats stats;
+
+    public ExecutorBeat(int timeSecs, int uptime, ExecutorStats stats) {
+        this.timeSecs = timeSecs;
+        this.uptime = uptime;
+        this.stats = stats;
+    }
+
+    public int getTimeSecs() {
+        return timeSecs;
+    }
+
+    public int getUptime() {
+        return uptime;
+    }
+
+    public ExecutorStats getStats() {
+        return stats;
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/IStateStorage.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/IStateStorage.java b/storm-core/src/jvm/org/apache/storm/cluster/IStateStorage.java
new file mode 100644
index 0000000..aa731ff
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/IStateStorage.java
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.curator.framework.state.ConnectionStateListener;
+import org.apache.storm.callback.ZKStateChangedCallback;
+import org.apache.zookeeper.data.ACL;
+
+/**
+ * StateStorage provides the API for the pluggable state store used by the
+ * Storm daemons. Data is stored in path/value format, and the store supports
+ * listing sub-paths at a given path.
+ * All data should be available across all nodes with eventual consistency.
+ *
+ * IMPORTANT NOTE:
+ * Heartbeats have different api calls used to interact with them. The root
+ * path (/) may or may not be the same as the root path for the other api calls.
+ *
+ * For example, performing these two calls:
+ *     set_data("/path", data, acls);
+ *     void set_worker_hb("/path", heartbeat, acls);
+ * may or may not cause a collision in "/path".
+ * Never use the same paths with the *_hb* methods as you do with the others.
+ */
+public interface IStateStorage {
+    
+    /**
+     * Registers a callback function that gets called when CuratorEvents happen.
+     * @param callback is a clojure IFn that accepts the type - translated to
+     * clojure keyword as in zookeeper - and the path: (callback type path)
+     * @return is an id that can be passed to unregister(...) to unregister the
+     * callback.
+     */
+    String register(ZKStateChangedCallback callback);
+
+    /**
+     * Unregisters a callback function that was registered with register(...).
+     * @param id is the String id that was returned from register(...).
+     */
+    void unregister(String id);
+
+    /**
+     * Path will be appended with a monotonically increasing integer, a new node
+     * will be created there, and data will be put at that node.
+     * @param path The path that the monotonically increasing integer suffix will
+     * be added to.
+     * @param data The data that will be written at the suffixed path's node.
+     * @param acls The acls to apply to the path. May be null.
+     * @return The path with the integer suffix appended.
+     */
+    String create_sequential(String path, byte[] data, List<ACL> acls);
+
+    /**
+     * Creates nodes for path and all its parents. Path elements are separated by
+     * a "/", as in *nix filesystem notation. Equivalent to mkdir -p in *nix.
+     * @param path The path to create, along with all its parents.
+     * @param acls The acls to apply to the path. May be null.
+     * @return path
+     */
+    void mkdirs(String path, List<ACL> acls);
+
+    /**
+     * Deletes the node at a given path, and any child nodes that may exist.
+     * @param path The path to delete
+     */
+    void delete_node(String path);
+
+    /**
+     * Creates an ephemeral node at path. Ephemeral nodes are destroyed
+     * by the store when the client disconnects.
+     * @param path The path where a node will be created.
+     * @param data The data to be written at the node.
+     * @param acls The acls to apply to the path. May be null.
+     */
+    void set_ephemeral_node(String path, byte[] data, List<ACL> acls);
+
+    /**
+     * Gets the 'version' of the node at a path. Optionally sets a watch
+     * on that node. The version should increase whenever a write happens.
+     * @param path The path to get the version of.
+     * @param watch Whether or not to set a watch on the path. Watched paths
+     * emit events which are consumed by functions registered with the
+     * register method. Very useful for catching updates to nodes.
+     * @return The integer version of this node.
+     */
+    Integer get_version(String path, boolean watch) throws Exception;
+
+    /**
+     * Check if a node exists and optionally set a watch on the path.
+     * @param path The path to check for the existence of a node.
+     * @param watch Whether or not to set a watch on the path. Watched paths
+     * emit events which are consumed by functions registered with the
+     * register method. Very useful for catching updates to nodes.
+     * @return Whether or not a node exists at path.
+     */
+    boolean node_exists(String path, boolean watch);
+
+    /**
+     * Get a list of paths of all the child nodes which exist immediately
+     * under path.
+     * @param path The path to look under
+     * @param watch Whether or not to set a watch on the path. Watched paths
+     * emit events which are consumed by functions registered with the
+     * register method. Very useful for catching updates to nodes.
+     * @return list of string paths under path.
+     */
+    List<String> get_children(String path, boolean watch);
+
+    /**
+     * Close the connection to the data store.
+     */
+    void close();
+
+    /**
+     * Set the value of the node at path to data.
+     * @param path The path whose node we want to set.
+     * @param data The data to put in the node.
+     * @param acls The acls to apply to the path. May be null.
+     */
+    void set_data(String path, byte[] data, List<ACL> acls);
+
+    /**
+     * Get the data from the node at path
+     * @param path The path to look under
+     * @param watch Whether or not to set a watch on the path. Watched paths
+     * emit events which are consumed by functions registered with the
+     * register method. Very useful for catching updates to nodes.
+     * @return The data at the node.
+     */
+    byte[] get_data(String path, boolean watch);
+
+    /**
+     * Get the data at the node along with its version. Data is returned
+     * in an Map with the keys data and version.
+     * @param path The path to look under
+     * @param watch Whether or not to set a watch on the path. Watched paths
+     * emit events which are consumed by functions registered with the
+     * register method. Very useful for catching updates to nodes.
+     * @return the data with a version
+     */
+    VersionedData<byte[]> get_data_with_version(String path, boolean watch);
+
+    /**
+     * Write a worker heartbeat at the path.
+     * @param path The path whose node we want to set.
+     * @param data The data to put in the node.
+     * @param acls The acls to apply to the path. May be null.
+     */
+    void set_worker_hb(String path, byte[] data, List<ACL> acls);
+
+    /**
+     * Get the heartbeat from the node at path
+     * @param path The path to look under
+     * @param watch Whether or not to set a watch on the path. Watched paths
+     * emit events which are consumed by functions registered with the
+     * register method. Very useful for catching updates to nodes.
+     * @return The heartbeat at the node.
+     */
+    byte[] get_worker_hb(String path, boolean watch);
+
+    /**
+     * Get a list of paths of all the child nodes which exist immediately
+     * under path. This is similar to get_children, but must be used for
+     * any nodes
+     * @param path The path to look under
+     * @param watch Whether or not to set a watch on the path. Watched paths
+     * emit events which are consumed by functions registered with the
+     * register method. Very useful for catching updates to nodes.
+     * @return list of string paths under path.
+     */
+    List<String> get_worker_hb_children(String path, boolean watch);
+
+    /**
+     * Deletes the heartbeat at a given path, and any child nodes that may exist.
+     * @param path The path to delete.
+     */
+    void delete_worker_hb(String path);
+
+    /**
+     * Add a StateStorageListener to the connection.
+     * @param listener A StateStorageListener to handle changing cluster state
+     * events.
+     */
+    void add_listener(final ConnectionStateListener listener);
+
+    /**
+     * Force consistency on a path. Any writes committed on the path before
+     * this call will be completely propagated when it returns.
+     * @param path The path to synchronize.
+     */
+    void sync_path(String path);
+
+    /**
+     * Allows us to delete the znodes within /storm/blobstore/key_name
+     * whose znodes start with the corresponding nimbusHostPortInfo
+     * @param path /storm/blobstore/key_name
+     * @param nimbusHostPortInfo Contains the host port information of
+     * a nimbus node.
+     */
+    void delete_node_blobstore(String path, String nimbusHostPortInfo);
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java b/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java
new file mode 100644
index 0000000..a6f07ed
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/IStormClusterState.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import org.apache.storm.generated.*;
+import org.apache.storm.nimbus.NimbusInfo;
+
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.Map;
+
+public interface IStormClusterState {
+    public List<String> assignments(Runnable callback);
+
+    public Assignment assignmentInfo(String stormId, Runnable callback);
+
+    public VersionedData<Assignment> assignmentInfoWithVersion(String stormId, Runnable callback);
+
+    public Integer assignmentVersion(String stormId, Runnable callback) throws Exception;
+
+    public List<String> blobstoreInfo(String blobKey);
+
+    public List nimbuses();
+
+    public void addNimbusHost(String nimbusId, NimbusSummary nimbusSummary);
+
+    public List<String> activeStorms();
+
+    public StormBase stormBase(String stormId, Runnable callback);
+
+    public ClusterWorkerHeartbeat getWorkerHeartbeat(String stormId, String node, Long port);
+
+    public List<ProfileRequest> getWorkerProfileRequests(String stormId, NodeInfo nodeInfo);
+
+    public List<ProfileRequest> getTopologyProfileRequests(String stormId);
+
+    public void setWorkerProfileRequest(String stormId, ProfileRequest profileRequest);
+
+    public void deleteTopologyProfileRequests(String stormId, ProfileRequest profileRequest);
+
+    public Map<ExecutorInfo, ExecutorBeat> executorBeats(String stormId, Map<List<Long>, NodeInfo> executorNodePort);
+
+    public List<String> supervisors(Runnable callback);
+
+    public SupervisorInfo supervisorInfo(String supervisorId); // returns nil if doesn't exist
+
+    public void setupHeatbeats(String stormId);
+
+    public void teardownHeartbeats(String stormId);
+
+    public void teardownTopologyErrors(String stormId);
+
+    public List<String> heartbeatStorms();
+
+    public List<String> errorTopologies();
+
+    public List<String> backpressureTopologies();
+
+    public void setTopologyLogConfig(String stormId, LogConfig logConfig);
+
+    public LogConfig topologyLogConfig(String stormId, Runnable cb);
+
+    public void workerHeartbeat(String stormId, String node, Long port, ClusterWorkerHeartbeat info);
+
+    public void removeWorkerHeartbeat(String stormId, String node, Long port);
+
+    public void supervisorHeartbeat(String supervisorId, SupervisorInfo info);
+
+    public void workerBackpressure(String stormId, String node, Long port, boolean on);
+
+    public boolean topologyBackpressure(String stormId, Runnable callback);
+
+    public void setupBackpressure(String stormId);
+
+    public void removeBackpressure(String stormId);
+
+    public void removeWorkerBackpressure(String stormId, String node, Long port);
+
+    public void activateStorm(String stormId, StormBase stormBase);
+
+    public void updateStorm(String stormId, StormBase newElems);
+
+    public void removeStormBase(String stormId);
+
+    public void setAssignment(String stormId, Assignment info);
+
+    public void setupBlobstore(String key, NimbusInfo nimbusInfo, Integer versionInfo);
+
+    public List<String> activeKeys();
+
+    public List<String> blobstore(Runnable callback);
+
+    public void removeStorm(String stormId);
+
+    public void removeBlobstoreKey(String blobKey);
+
+    public void removeKeyVersion(String blobKey);
+
+    public void reportError(String stormId, String componentId, String node, Long port, Throwable error);
+
+    public List<ErrorInfo> errors(String stormId, String componentId);
+
+    public ErrorInfo lastError(String stormId, String componentId);
+
+    public void setCredentials(String stormId, Credentials creds, Map topoConf) throws NoSuchAlgorithmException;
+
+    public Credentials credentials(String stormId, Runnable callback);
+
+    public void disconnect();
+
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/StateStorageFactory.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/StateStorageFactory.java b/storm-core/src/jvm/org/apache/storm/cluster/StateStorageFactory.java
new file mode 100644
index 0000000..0929750
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/StateStorageFactory.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.zookeeper.data.ACL;
+
+public interface StateStorageFactory {
+
+    IStateStorage mkStore(Map config, Map auth_conf, List<ACL> acls, ClusterStateContext context);
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java b/storm-core/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java
new file mode 100644
index 0000000..972d778
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java
@@ -0,0 +1,720 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.state.*;
+import org.apache.curator.framework.state.ConnectionState;
+import org.apache.storm.callback.ZKStateChangedCallback;
+import org.apache.storm.generated.*;
+import org.apache.storm.nimbus.NimbusInfo;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Utils;
+import org.apache.storm.zookeeper.Zookeeper;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.ACL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.NoSuchAlgorithmException;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class StormClusterStateImpl implements IStormClusterState {
+
+    private static Logger LOG = LoggerFactory.getLogger(StormClusterStateImpl.class);
+
+    private IStateStorage stateStorage;
+
+    private ConcurrentHashMap<String, Runnable> assignmentInfoCallback;
+    private ConcurrentHashMap<String, Runnable> assignmentInfoWithVersionCallback;
+    private ConcurrentHashMap<String, Runnable> assignmentVersionCallback;
+    private AtomicReference<Runnable> supervisorsCallback;
+    // we want to reigister a topo directory getChildren callback for all workers of this dir
+    private ConcurrentHashMap<String, Runnable> backPressureCallback;
+    private AtomicReference<Runnable> assignmentsCallback;
+    private ConcurrentHashMap<String, Runnable> stormBaseCallback;
+    private AtomicReference<Runnable> blobstoreCallback;
+    private ConcurrentHashMap<String, Runnable> credentialsCallback;
+    private ConcurrentHashMap<String, Runnable> logConfigCallback;
+
+    private List<ACL> acls;
+    private String stateId;
+    private boolean solo;
+
+    public StormClusterStateImpl(IStateStorage StateStorage, List<ACL> acls, ClusterStateContext context, boolean solo) throws Exception {
+
+        this.stateStorage = StateStorage;
+        this.solo = solo;
+        this.acls = acls;
+
+        assignmentInfoCallback = new ConcurrentHashMap<>();
+        assignmentInfoWithVersionCallback = new ConcurrentHashMap<>();
+        assignmentVersionCallback = new ConcurrentHashMap<>();
+        supervisorsCallback = new AtomicReference<>();
+        backPressureCallback = new ConcurrentHashMap<>();
+        assignmentsCallback = new AtomicReference<>();
+        stormBaseCallback = new ConcurrentHashMap<>();
+        credentialsCallback = new ConcurrentHashMap<>();
+        logConfigCallback = new ConcurrentHashMap<>();
+        blobstoreCallback = new AtomicReference<>();
+
+        stateId = this.stateStorage.register(new ZKStateChangedCallback() {
+
+            public void changed(Watcher.Event.EventType type, String path) {
+                List<String> toks = Zookeeper.tokenizePath(path);
+                int size = toks.size();
+                if (size >= 1) {
+                    String root = toks.get(0);
+                    if (root.equals(ClusterUtils.ASSIGNMENTS_ROOT)) {
+                        if (size == 1) {
+                            // set null and get the old value
+                            issueCallback(assignmentsCallback);
+                        } else {
+                            issueMapCallback(assignmentInfoCallback, toks.get(1));
+                            issueMapCallback(assignmentVersionCallback, toks.get(1));
+                            issueMapCallback(assignmentInfoWithVersionCallback, toks.get(1));
+                        }
+
+                    } else if (root.equals(ClusterUtils.SUPERVISORS_ROOT)) {
+                        issueCallback(supervisorsCallback);
+                    } else if (root.equals(ClusterUtils.BLOBSTORE_ROOT)) {
+                        issueCallback(blobstoreCallback);
+                    } else if (root.equals(ClusterUtils.STORMS_ROOT) && size > 1) {
+                        issueMapCallback(stormBaseCallback, toks.get(1));
+                    } else if (root.equals(ClusterUtils.CREDENTIALS_ROOT) && size > 1) {
+                        issueMapCallback(credentialsCallback, toks.get(1));
+                    } else if (root.equals(ClusterUtils.LOGCONFIG_ROOT) && size > 1) {
+                        issueMapCallback(logConfigCallback, toks.get(1));
+                    } else if (root.equals(ClusterUtils.BACKPRESSURE_ROOT) && size > 1) {
+                        issueMapCallback(backPressureCallback, toks.get(1));
+                    } else {
+                        LOG.error("{} Unknown callback for subtree {}", new RuntimeException("Unknown callback for this path"), path);
+                        Runtime.getRuntime().exit(30);
+                    }
+
+                }
+
+                return;
+            }
+
+        });
+
+        String[] pathlist = { ClusterUtils.ASSIGNMENTS_SUBTREE, 
+                              ClusterUtils.STORMS_SUBTREE, 
+                              ClusterUtils.SUPERVISORS_SUBTREE, 
+                              ClusterUtils.WORKERBEATS_SUBTREE,
+                              ClusterUtils.ERRORS_SUBTREE, 
+                              ClusterUtils.BLOBSTORE_SUBTREE, 
+                              ClusterUtils.NIMBUSES_SUBTREE, 
+                              ClusterUtils.LOGCONFIG_SUBTREE,
+                              ClusterUtils.BACKPRESSURE_SUBTREE };
+        for (String path : pathlist) {
+            this.stateStorage.mkdirs(path, acls);
+        }
+
+    }
+
+    protected void issueCallback(AtomicReference<Runnable> cb) {
+        Runnable callback = cb.getAndSet(null);
+        if (callback != null)
+            callback.run();
+    }
+
+    protected void issueMapCallback(ConcurrentHashMap<String, Runnable> callbackConcurrentHashMap, String key) {
+        Runnable callback = callbackConcurrentHashMap.remove(key);
+        if (callback != null)
+            callback.run();
+    }
+
+    @Override
+    public List<String> assignments(Runnable callback) {
+        if (callback != null) {
+            assignmentsCallback.set(callback);
+        }
+        return stateStorage.get_children(ClusterUtils.ASSIGNMENTS_SUBTREE, callback != null);
+    }
+
+    @Override
+    public Assignment assignmentInfo(String stormId, Runnable callback) {
+        if (callback != null) {
+            assignmentInfoCallback.put(stormId, callback);
+        }
+        byte[] serialized = stateStorage.get_data(ClusterUtils.assignmentPath(stormId), callback != null);
+        return ClusterUtils.maybeDeserialize(serialized, Assignment.class);
+    }
+
+    @Override
+    public VersionedData<Assignment> assignmentInfoWithVersion(String stormId, Runnable callback) {
+        if (callback != null) {
+            assignmentInfoWithVersionCallback.put(stormId, callback);
+        }
+        Assignment assignment = null;
+        Integer version = 0;
+        VersionedData<byte[]> dataWithVersion = stateStorage.get_data_with_version(ClusterUtils.assignmentPath(stormId), callback != null);
+        if (dataWithVersion != null) {
+            assignment = ClusterUtils.maybeDeserialize(dataWithVersion.getData(), Assignment.class);
+            version = dataWithVersion.getVersion();
+        }
+        return new VersionedData<Assignment>(version, assignment);
+    }
+
+    @Override
+    public Integer assignmentVersion(String stormId, Runnable callback) throws Exception {
+        if (callback != null) {
+            assignmentVersionCallback.put(stormId, callback);
+        }
+        return stateStorage.get_version(ClusterUtils.assignmentPath(stormId), callback != null);
+    }
+
+    // blobstore state
+    @Override
+    public List<String> blobstoreInfo(String blobKey) {
+        String path = ClusterUtils.blobstorePath(blobKey);
+        stateStorage.sync_path(path);
+        return stateStorage.get_children(path, false);
+    }
+
+    @Override
+    public List nimbuses() {
+        List<NimbusSummary> nimbusSummaries = new ArrayList<>();
+        List<String> nimbusIds = stateStorage.get_children(ClusterUtils.NIMBUSES_SUBTREE, false);
+        for (String nimbusId : nimbusIds) {
+            byte[] serialized = stateStorage.get_data(ClusterUtils.nimbusPath(nimbusId), false);
+            NimbusSummary nimbusSummary = ClusterUtils.maybeDeserialize(serialized, NimbusSummary.class);
+            nimbusSummaries.add(nimbusSummary);
+        }
+        return nimbusSummaries;
+    }
+
+    @Override
+    public void addNimbusHost(final String nimbusId, final NimbusSummary nimbusSummary) {
+        // explicit delete for ephmeral node to ensure this session creates the entry.
+        stateStorage.delete_node(ClusterUtils.nimbusPath(nimbusId));
+        stateStorage.add_listener(new ConnectionStateListener() {
+            @Override
+            public void stateChanged(CuratorFramework curatorFramework, ConnectionState connectionState) {
+                LOG.info("Connection state listener invoked, zookeeper connection state has changed to {}", connectionState);
+                if (connectionState.equals(ConnectionState.RECONNECTED)) {
+                    LOG.info("Connection state has changed to reconnected so setting nimbuses entry one more time");
+                    // explicit delete for ephmeral node to ensure this session creates the entry.
+                    stateStorage.delete_node(ClusterUtils.nimbusPath(nimbusId));
+                    stateStorage.set_ephemeral_node(ClusterUtils.nimbusPath(nimbusId), Utils.serialize(nimbusSummary), acls);
+                }
+
+            }
+        });
+
+        stateStorage.set_ephemeral_node(ClusterUtils.nimbusPath(nimbusId), Utils.serialize(nimbusSummary), acls);
+    }
+
+    @Override
+    public List<String> activeStorms() {
+        return stateStorage.get_children(ClusterUtils.STORMS_SUBTREE, false);
+    }
+
+    @Override
+    public StormBase stormBase(String stormId, Runnable callback) {
+        if (callback != null) {
+            stormBaseCallback.put(stormId, callback);
+        }
+        return ClusterUtils.maybeDeserialize(stateStorage.get_data(ClusterUtils.stormPath(stormId), callback != null), StormBase.class);
+    }
+
+    @Override
+    public ClusterWorkerHeartbeat getWorkerHeartbeat(String stormId, String node, Long port) {
+        byte[] bytes = stateStorage.get_worker_hb(ClusterUtils.workerbeatPath(stormId, node, port), false);
+        return ClusterUtils.maybeDeserialize(bytes, ClusterWorkerHeartbeat.class);
+
+    }
+
+    @Override
+    public List<ProfileRequest> getWorkerProfileRequests(String stormId, NodeInfo nodeInfo) {
+        List<ProfileRequest> requests = new ArrayList<>();
+        List<ProfileRequest> profileRequests = getTopologyProfileRequests(stormId);
+        for (ProfileRequest profileRequest : profileRequests) {
+            NodeInfo nodeInfo1 = profileRequest.get_nodeInfo();
+            if (nodeInfo1.equals(nodeInfo))
+                requests.add(profileRequest);
+        }
+        return requests;
+    }
+
+    @Override
+    public List<ProfileRequest> getTopologyProfileRequests(String stormId) {
+        List<ProfileRequest> profileRequests = new ArrayList<>();
+        String path = ClusterUtils.profilerConfigPath(stormId);
+        if (stateStorage.node_exists(path, false)) {
+            List<String> strs = stateStorage.get_children(path, false);
+            for (String str : strs) {
+                String childPath = path + ClusterUtils.ZK_SEPERATOR + str;
+                byte[] raw = stateStorage.get_data(childPath, false);
+                ProfileRequest request = ClusterUtils.maybeDeserialize(raw, ProfileRequest.class);
+                if (request != null)
+                    profileRequests.add(request);
+            }
+        }
+        return profileRequests;
+    }
+
+    @Override
+    public void setWorkerProfileRequest(String stormId, ProfileRequest profileRequest) {
+        ProfileAction profileAction = profileRequest.get_action();
+        String host = profileRequest.get_nodeInfo().get_node();
+        Long port = profileRequest.get_nodeInfo().get_port_iterator().next();
+        String path = ClusterUtils.profilerConfigPath(stormId, host, port, profileAction);
+        stateStorage.set_data(path, Utils.serialize(profileRequest), acls);
+    }
+
+    @Override
+    public void deleteTopologyProfileRequests(String stormId, ProfileRequest profileRequest) {
+        ProfileAction profileAction = profileRequest.get_action();
+        String host = profileRequest.get_nodeInfo().get_node();
+        Long port = profileRequest.get_nodeInfo().get_port_iterator().next();
+        String path = ClusterUtils.profilerConfigPath(stormId, host, port, profileAction);
+        stateStorage.delete_node(path);
+    }
+
+    /**
+     * need to take executor->node+port in explicitly so that we don't run into a situation where a long dead worker with a skewed clock overrides all the
+     * timestamps. By only checking heartbeats with an assigned node+port, and only reading executors from that heartbeat that are actually assigned, we avoid
+     * situations like that
+     * 
+     * @param stormId
+     * @param executorNodePort
+     * @return
+     */
+    @Override
+    public Map<ExecutorInfo, ExecutorBeat> executorBeats(String stormId, Map<List<Long>, NodeInfo> executorNodePort) {
+        Map<ExecutorInfo, ExecutorBeat> executorWhbs = new HashMap<>();
+
+        Map<NodeInfo, List<List<Long>>> nodePortExecutors = Utils.reverseMap(executorNodePort);
+
+        for (Map.Entry<NodeInfo, List<List<Long>>> entry : nodePortExecutors.entrySet()) {
+
+            String node = entry.getKey().get_node();
+            Long port = entry.getKey().get_port_iterator().next();
+            ClusterWorkerHeartbeat whb = getWorkerHeartbeat(stormId, node, port);
+            List<ExecutorInfo> executorInfoList = new ArrayList<>();
+            for (List<Long> list : entry.getValue()) {
+                executorInfoList.add(new ExecutorInfo(list.get(0).intValue(), list.get(list.size() - 1).intValue()));
+            }
+            if (whb != null)
+                executorWhbs.putAll(ClusterUtils.convertExecutorBeats(executorInfoList, whb));
+        }
+        return executorWhbs;
+    }
+
+    @Override
+    public List<String> supervisors(Runnable callback) {
+        if (callback != null) {
+            supervisorsCallback.set(callback);
+        }
+        return stateStorage.get_children(ClusterUtils.SUPERVISORS_SUBTREE, callback != null);
+    }
+
+    @Override
+    public SupervisorInfo supervisorInfo(String supervisorId) {
+        String path = ClusterUtils.supervisorPath(supervisorId);
+        return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), SupervisorInfo.class);
+    }
+
+    @Override
+    public void setupHeatbeats(String stormId) {
+        stateStorage.mkdirs(ClusterUtils.workerbeatStormRoot(stormId), acls);
+    }
+
+    @Override
+    public void teardownHeartbeats(String stormId) {
+        try {
+            stateStorage.delete_worker_hb(ClusterUtils.workerbeatStormRoot(stormId));
+        } catch (Exception e) {
+            if (Utils.exceptionCauseIsInstanceOf(KeeperException.class, e)) {
+                // do nothing
+                LOG.warn("Could not teardown heartbeats for {}.", stormId);
+            } else {
+                throw e;
+            }
+        }
+    }
+
+    @Override
+    public void teardownTopologyErrors(String stormId) {
+        try {
+            stateStorage.delete_node(ClusterUtils.errorStormRoot(stormId));
+        } catch (Exception e) {
+            if (Utils.exceptionCauseIsInstanceOf(KeeperException.class, e)) {
+                // do nothing
+                LOG.warn("Could not teardown errors for {}.", stormId);
+            } else {
+                throw e;
+            }
+        }
+    }
+
+    @Override
+    public List<String> heartbeatStorms() {
+        return stateStorage.get_worker_hb_children(ClusterUtils.WORKERBEATS_SUBTREE, false);
+    }
+
+    @Override
+    public List<String> errorTopologies() {
+        return stateStorage.get_children(ClusterUtils.ERRORS_SUBTREE, false);
+    }
+
+    @Override
+    public List<String> backpressureTopologies() {
+        return stateStorage.get_children(ClusterUtils.BACKPRESSURE_SUBTREE, false);
+    }
+
+    @Override
+    public void setTopologyLogConfig(String stormId, LogConfig logConfig) {
+        stateStorage.set_data(ClusterUtils.logConfigPath(stormId), Utils.serialize(logConfig), acls);
+    }
+
+    @Override
+    public LogConfig topologyLogConfig(String stormId, Runnable cb) {
+        if (cb != null){
+            logConfigCallback.put(stormId, cb);
+        }
+        String path = ClusterUtils.logConfigPath(stormId);
+        return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, cb != null), LogConfig.class);
+    }
+
+    @Override
+    public void workerHeartbeat(String stormId, String node, Long port, ClusterWorkerHeartbeat info) {
+        if (info != null) {
+            String path = ClusterUtils.workerbeatPath(stormId, node, port);
+            stateStorage.set_worker_hb(path, Utils.serialize(info), acls);
+        }
+    }
+
+    @Override
+    public void removeWorkerHeartbeat(String stormId, String node, Long port) {
+        String path = ClusterUtils.workerbeatPath(stormId, node, port);
+        stateStorage.delete_worker_hb(path);
+    }
+
+    @Override
+    public void supervisorHeartbeat(String supervisorId, SupervisorInfo info) {
+        String path = ClusterUtils.supervisorPath(supervisorId);
+        stateStorage.set_ephemeral_node(path, Utils.serialize(info), acls);
+    }
+
+    /**
+     * if znode exists and to be not on?, delete; if exists and on?, do nothing; if not exists and to be on?, create; if not exists and not on?, do nothing;
+     * 
+     * @param stormId
+     * @param node
+     * @param port
+     * @param on
+     */
+    @Override
+    public void workerBackpressure(String stormId, String node, Long port, boolean on) {
+        String path = ClusterUtils.backpressurePath(stormId, node, port);
+        boolean existed = stateStorage.node_exists(path, false);
+        if (existed) {
+            if (on == false)
+                stateStorage.delete_node(path);
+
+        } else {
+            if (on == true) {
+                stateStorage.set_ephemeral_node(path, null, acls);
+            }
+        }
+    }
+
+    /**
+     * Check whether a topology is in throttle-on status or not:
+     * if the backpresure/storm-id dir is not empty, this topology has throttle-on, otherwise throttle-off.
+     * 
+     * @param stormId
+     * @param callback
+     * @return
+     */
+    @Override
+    public boolean topologyBackpressure(String stormId, Runnable callback) {
+        if (callback != null) {
+            backPressureCallback.put(stormId, callback);
+        }
+        String path = ClusterUtils.backpressureStormRoot(stormId);
+        List<String> childrens = null;
+        if(stateStorage.node_exists(path, false)) {
+            childrens = stateStorage.get_children(path, callback != null);
+        } else {
+            childrens = new ArrayList<>();
+        }
+        return childrens.size() > 0;
+
+    }
+
+    @Override
+    public void setupBackpressure(String stormId) {
+        stateStorage.mkdirs(ClusterUtils.backpressureStormRoot(stormId), acls);
+    }
+
+    @Override
+    public void removeBackpressure(String stormId) {
+        try {
+            stateStorage.delete_node(ClusterUtils.backpressureStormRoot(stormId));
+        } catch (Exception e) {
+            if (Utils.exceptionCauseIsInstanceOf(KeeperException.class, e)) {
+                // do nothing
+                LOG.warn("Could not teardown backpressure node for {}.", stormId);
+            } else {
+                throw e;
+            }
+        }
+    }
+
+    @Override
+    public void removeWorkerBackpressure(String stormId, String node, Long port) {
+        String path = ClusterUtils.backpressurePath(stormId, node, port);
+        boolean existed = stateStorage.node_exists(path, false);
+        if (existed) {
+            stateStorage.delete_node(path);
+        }
+    }
+
+    @Override
+    public void activateStorm(String stormId, StormBase stormBase) {
+        String path = ClusterUtils.stormPath(stormId);
+        stateStorage.set_data(path, Utils.serialize(stormBase), acls);
+    }
+
+    /**
+     * To update this function due to APersistentMap/APersistentSet is clojure's structure
+     * 
+     * @param stormId
+     * @param newElems
+     */
+    @Override
+    public void updateStorm(String stormId, StormBase newElems) {
+
+        StormBase stormBase = stormBase(stormId, null);
+        if (stormBase.get_component_executors() != null) {
+
+            Map<String, Integer> newComponentExecutors = new HashMap<>();
+            Map<String, Integer> componentExecutors = newElems.get_component_executors();
+            // componentExecutors maybe be APersistentMap, which don't support "put"
+            for (Map.Entry<String, Integer> entry : componentExecutors.entrySet()) {
+                newComponentExecutors.put(entry.getKey(), entry.getValue());
+            }
+            for (Map.Entry<String, Integer> entry : stormBase.get_component_executors().entrySet()) {
+                if (!componentExecutors.containsKey(entry.getKey())) {
+                    newComponentExecutors.put(entry.getKey(), entry.getValue());
+                }
+            }
+            if (newComponentExecutors.size() > 0)
+                newElems.set_component_executors(newComponentExecutors);
+        }
+
+        Map<String, DebugOptions> ComponentDebug = new HashMap<>();
+        Map<String, DebugOptions> oldComponentDebug = stormBase.get_component_debug();
+
+        Map<String, DebugOptions> newComponentDebug = newElems.get_component_debug();
+        /// oldComponentDebug.keySet()/ newComponentDebug.keySet() maybe be APersistentSet, which don't support addAll
+        Set<String> debugOptionsKeys = new HashSet<>();
+        debugOptionsKeys.addAll(oldComponentDebug.keySet());
+        debugOptionsKeys.addAll(newComponentDebug.keySet());
+        for (String key : debugOptionsKeys) {
+            boolean enable = false;
+            double samplingpct = 0;
+            if (oldComponentDebug.containsKey(key)) {
+                enable = oldComponentDebug.get(key).is_enable();
+                samplingpct = oldComponentDebug.get(key).get_samplingpct();
+            }
+            if (newComponentDebug.containsKey(key)) {
+                enable = newComponentDebug.get(key).is_enable();
+                samplingpct += newComponentDebug.get(key).get_samplingpct();
+            }
+            DebugOptions debugOptions = new DebugOptions();
+            debugOptions.set_enable(enable);
+            debugOptions.set_samplingpct(samplingpct);
+            ComponentDebug.put(key, debugOptions);
+        }
+        if (ComponentDebug.size() > 0) {
+            newElems.set_component_debug(ComponentDebug);
+        }
+
+        if (StringUtils.isBlank(newElems.get_name())) {
+            newElems.set_name(stormBase.get_name());
+        }
+        if (newElems.get_status() == null) {
+            newElems.set_status(stormBase.get_status());
+        }
+        if (newElems.get_num_workers() == 0) {
+            newElems.set_num_workers(stormBase.get_num_workers());
+        }
+        if (newElems.get_launch_time_secs() == 0) {
+            newElems.set_launch_time_secs(stormBase.get_launch_time_secs());
+        }
+        if (StringUtils.isBlank(newElems.get_owner())) {
+            newElems.set_owner(stormBase.get_owner());
+        }
+        if (newElems.get_topology_action_options() == null) {
+            newElems.set_topology_action_options(stormBase.get_topology_action_options());
+        }
+        if (newElems.get_status() == null) {
+            newElems.set_status(stormBase.get_status());
+        }
+        stateStorage.set_data(ClusterUtils.stormPath(stormId), Utils.serialize(newElems), acls);
+    }
+
+    @Override
+    public void removeStormBase(String stormId) {
+        stateStorage.delete_node(ClusterUtils.stormPath(stormId));
+    }
+
+    @Override
+    public void setAssignment(String stormId, Assignment info) {
+        stateStorage.set_data(ClusterUtils.assignmentPath(stormId), Utils.serialize(info), acls);
+    }
+
+    @Override
+    public void setupBlobstore(String key, NimbusInfo nimbusInfo, Integer versionInfo) {
+        String path = ClusterUtils.blobstorePath(key) + ClusterUtils.ZK_SEPERATOR + nimbusInfo.toHostPortString() + "-" + versionInfo;
+        LOG.info("set-path: {}", path);
+        stateStorage.mkdirs(ClusterUtils.blobstorePath(key), acls);
+        stateStorage.delete_node_blobstore(ClusterUtils.blobstorePath(key), nimbusInfo.toHostPortString());
+        stateStorage.set_ephemeral_node(path, null, acls);
+    }
+
+    @Override
+    public List<String> activeKeys() {
+        return stateStorage.get_children(ClusterUtils.BLOBSTORE_SUBTREE, false);
+    }
+
+    // blobstore state
+    @Override
+    public List<String> blobstore(Runnable callback) {
+        if (callback != null) {
+            blobstoreCallback.set(callback);
+        }
+        stateStorage.sync_path(ClusterUtils.BLOBSTORE_SUBTREE);
+        return stateStorage.get_children(ClusterUtils.BLOBSTORE_SUBTREE, callback != null);
+
+    }
+
+    @Override
+    public void removeStorm(String stormId) {
+        stateStorage.delete_node(ClusterUtils.assignmentPath(stormId));
+        stateStorage.delete_node(ClusterUtils.credentialsPath(stormId));
+        stateStorage.delete_node(ClusterUtils.logConfigPath(stormId));
+        stateStorage.delete_node(ClusterUtils.profilerConfigPath(stormId));
+        removeStormBase(stormId);
+    }
+
+    @Override
+    public void removeBlobstoreKey(String blobKey) {
+        LOG.debug("remove key {}", blobKey);
+        stateStorage.delete_node(ClusterUtils.blobstorePath(blobKey));
+    }
+
+    @Override
+    public void removeKeyVersion(String blobKey) {
+        stateStorage.delete_node(ClusterUtils.blobstoreMaxKeySequenceNumberPath(blobKey));
+    }
+
+    @Override
+    public void reportError(String stormId, String componentId, String node, Long port, Throwable error) {
+
+        String path = ClusterUtils.errorPath(stormId, componentId);
+        String lastErrorPath = ClusterUtils.lastErrorPath(stormId, componentId);
+        ErrorInfo errorInfo = new ErrorInfo(ClusterUtils.stringifyError(error), Time.currentTimeSecs());
+        errorInfo.set_host(node);
+        errorInfo.set_port(port.intValue());
+        byte[] serData = Utils.serialize(errorInfo);
+        stateStorage.mkdirs(path, acls);
+        stateStorage.create_sequential(path + ClusterUtils.ZK_SEPERATOR + "e", serData, acls);
+        stateStorage.set_data(lastErrorPath, serData, acls);
+        List<String> childrens = stateStorage.get_children(path, false);
+
+        Collections.sort(childrens, new Comparator<String>() {
+            public int compare(String arg0, String arg1) {
+                return Long.compare(Long.parseLong(arg0.substring(1)), Long.parseLong(arg1.substring(1)));
+            }
+        });
+
+        while (childrens.size() > 10) {
+            stateStorage.delete_node(path + ClusterUtils.ZK_SEPERATOR + childrens.remove(0));
+        }
+    }
+
+    @Override
+    public List<ErrorInfo> errors(String stormId, String componentId) {
+        List<ErrorInfo> errorInfos = new ArrayList<>();
+        String path = ClusterUtils.errorPath(stormId, componentId);
+        if (stateStorage.node_exists(path, false)) {
+            List<String> childrens = stateStorage.get_children(path, false);
+            for (String child : childrens) {
+                String childPath = path + ClusterUtils.ZK_SEPERATOR + child;
+                ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(childPath, false), ErrorInfo.class);
+                if (errorInfo != null)
+                    errorInfos.add(errorInfo);
+            }
+        }
+        Collections.sort(errorInfos, new Comparator<ErrorInfo>() {
+            public int compare(ErrorInfo arg0, ErrorInfo arg1) {
+                return Integer.compare(arg1.get_error_time_secs(), arg0.get_error_time_secs());
+            }
+        });
+
+        return errorInfos;
+    }
+
+    @Override
+    public ErrorInfo lastError(String stormId, String componentId) {
+
+        String path = ClusterUtils.lastErrorPath(stormId, componentId);
+        if (stateStorage.node_exists(path, false)) {
+            ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(path, false), ErrorInfo.class);
+            return errorInfo;
+        }
+
+        return null;
+    }
+
+    @Override
+    public void setCredentials(String stormId, Credentials creds, Map topoConf) throws NoSuchAlgorithmException {
+        List<ACL> aclList = ClusterUtils.mkTopoOnlyAcls(topoConf);
+        String path = ClusterUtils.credentialsPath(stormId);
+        stateStorage.set_data(path, Utils.serialize(creds), aclList);
+
+    }
+
+    @Override
+    public Credentials credentials(String stormId, Runnable callback) {
+        if (callback != null) {
+            credentialsCallback.put(stormId, callback);
+        }
+        String path = ClusterUtils.credentialsPath(stormId);
+        return ClusterUtils.maybeDeserialize(stateStorage.get_data(path, callback != null), Credentials.class);
+
+    }
+
+    @Override
+    public void disconnect() {
+        stateStorage.unregister(stateId);
+        if (solo)
+            stateStorage.close();
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/318ab5f3/storm-core/src/jvm/org/apache/storm/cluster/VersionedData.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/cluster/VersionedData.java b/storm-core/src/jvm/org/apache/storm/cluster/VersionedData.java
new file mode 100644
index 0000000..3de2a88
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/cluster/VersionedData.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.cluster;
+
+public class VersionedData<D> {
+    private final int version;
+    private final D data;
+    
+    public VersionedData(int version, D data) {
+        this.version = version;
+        this.data = data;
+    }
+    
+    public int getVersion() {
+        return version;
+    }
+    
+    public D getData() {
+        return data;
+    }
+}