You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2017/10/10 18:48:24 UTC

[2/8] storm git commit: STORM-2438: added in rebalance changes to support RAS

http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/LocalDownloadedResource.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalDownloadedResource.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalDownloadedResource.java
index f019374..2d2e879 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocalDownloadedResource.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalDownloadedResource.java
@@ -15,59 +15,31 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.storm.localizer;
 
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 
 import org.apache.storm.generated.LocalAssignment;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * Used for accounting to keep track of who is waiting for specific resources to be downloaded.
+ */
 public class LocalDownloadedResource {
     private static final Logger LOG = LoggerFactory.getLogger(LocalDownloadedResource.class);
-    private static class PortNAssignment {
-        private final int _port;
-        private final LocalAssignment _assignment;
-        
-        public PortNAssignment(int port, LocalAssignment assignment) {
-            _port = port;
-            _assignment = assignment;
-        }
-        
-        @Override
-        public boolean equals(Object other) {
-            if (!(other instanceof PortNAssignment)) {
-                return false;
-            }
-            PortNAssignment pna = (PortNAssignment) other;
-            return pna._port == _port && _assignment.equals(pna._assignment); 
-        }
-        
-        @Override
-        public int hashCode() {
-            return (17 * _port) + _assignment.hashCode();
-        }
-        
-        @Override
-        public String toString() {
-            return "{"+ _port + " " + _assignment +"}";
-        }
-    }
-    private final CompletableFuture<Void> _pending;
-    private final Set<PortNAssignment> _references;
-    private boolean _isDone;
+    private final CompletableFuture<Void> pending;
+    private final Set<PortAndAssignment> references;
+    private boolean isDone;
     
     
     public LocalDownloadedResource(CompletableFuture<Void> pending) {
-        _pending = pending;
-        _references = new HashSet<>();
-        _isDone = false;
+        this.pending = pending;
+        references = new HashSet<>();
+        isDone = false;
     }
 
     /**
@@ -77,11 +49,11 @@ public class LocalDownloadedResource {
      * @return a future that can be used to track it being downloaded.
      */
     public synchronized CompletableFuture<Void> reserve(int port, LocalAssignment la) {
-        PortNAssignment pna = new PortNAssignment(port, la);
-        if (!_references.add(pna)) {
-            LOG.warn("Resources {} already reserved {} for this topology", pna, _references);
+        PortAndAssignment pna = new PortAndAssignment(port, la);
+        if (!references.add(pna)) {
+            LOG.warn("Resources {} already reserved {} for this topology", pna, references);
         }
-        return _pending;
+        return pending;
     }
     
     /**
@@ -91,10 +63,10 @@ public class LocalDownloadedResource {
      * @return true if the port was being counted else false
      */
     public synchronized boolean release(int port, LocalAssignment la) {
-        PortNAssignment pna = new PortNAssignment(port, la);
-        boolean ret = _references.remove(pna);
-        if (ret && _references.isEmpty()) {
-            _isDone = true;
+        PortAndAssignment pna = new PortAndAssignment(port, la);
+        boolean ret = references.remove(pna);
+        if (ret && references.isEmpty()) {
+            isDone = true;
         }
         return ret;
     }
@@ -104,11 +76,11 @@ public class LocalDownloadedResource {
      * @return true if it is done else false
      */
     public synchronized boolean isDone() {
-        return _isDone;
+        return isDone;
     }
 
     @Override
     public String toString() {
-        return _references.toString();
+        return references.toString();
     }
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
index b6b5cd1..7241976 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
@@ -127,4 +127,17 @@ public class LocalizedResource {
     return System.nanoTime();
   }
 
+  @Override
+  public boolean equals(Object other) {
+    if (other instanceof LocalizedResource) {
+        LocalizedResource l = (LocalizedResource)other;
+        return _key.equals(l._key) && _uncompressed == l._uncompressed && _localPath.equals(l._localPath);
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+     return _key.hashCode() + Boolean.hashCode(_uncompressed) + _localPath.hashCode();
+  }
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
index 9f42b47..826bf98 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
@@ -17,6 +17,9 @@
  */
 package org.apache.storm.localizer;
 
+import com.google.common.annotations.VisibleForTesting;
+import java.nio.file.Path;
+import java.util.concurrent.ConcurrentHashMap;
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -29,112 +32,325 @@ import java.util.Iterator;
 import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
+
 /**
  * A set of resources that we can look at to see which ones we retain and which ones should be
  * removed.
  */
 public class LocalizedResourceRetentionSet {
-  public static final Logger LOG = LoggerFactory.getLogger(LocalizedResourceRetentionSet.class);
-  private long _delSize;
-  private long _currentSize;
-  // targetSize in Bytes
-  private long _targetSize;
-  private final SortedMap<LocalizedResource, LocalizedResourceSet> _noReferences;
-
-  LocalizedResourceRetentionSet(long targetSize) {
-    this(targetSize, new LRUComparator());
-  }
-
-  LocalizedResourceRetentionSet(long targetSize, Comparator<? super LocalizedResource> cmp) {
-    this(targetSize, new TreeMap<LocalizedResource, LocalizedResourceSet>(cmp));
-  }
-
-  LocalizedResourceRetentionSet(long targetSize,
-                                SortedMap<LocalizedResource, LocalizedResourceSet> retain) {
-    this._noReferences = retain;
-    this._targetSize = targetSize;
-  }
-
-  // for testing
-  protected int getSizeWithNoReferences() {
-    return _noReferences.size();
-  }
-
-  protected void addResourcesForSet(Iterator<LocalizedResource> setIter, LocalizedResourceSet set) {
-    for (Iterator<LocalizedResource> iter = setIter; setIter.hasNext(); ) {
-      LocalizedResource lrsrc = iter.next();
-      _currentSize += lrsrc.getSize();
-      if (lrsrc.getRefCount() > 0) {
-        // always retain resources in use
-        continue;
-      }
-      LOG.debug("adding {} to be checked for cleaning", lrsrc.getKey());
-      _noReferences.put(lrsrc, set);
-    }
-  }
-
-  public void addResources(LocalizedResourceSet set) {
-    addResourcesForSet(set.getLocalFilesIterator(), set);
-    addResourcesForSet(set.getLocalArchivesIterator(), set);
-  }
-
-  public void cleanup() {
-    LOG.debug("cleanup target size: {} current size is: {}", _targetSize, _currentSize);
-    for (Iterator<Map.Entry<LocalizedResource, LocalizedResourceSet>> i =
-             _noReferences.entrySet().iterator();
-         _currentSize - _delSize > _targetSize && i.hasNext();) {
-      Map.Entry<LocalizedResource, LocalizedResourceSet> rsrc = i.next();
-      LocalizedResource resource = rsrc.getKey();
-      LocalizedResourceSet set = rsrc.getValue();
-      if (resource != null && set.remove(resource)) {
-        if (deleteResource(resource)) {
-          _delSize += resource.getSize();
-          LOG.info("deleting: " + resource.getFilePath() + " size of: " + resource.getSize());
-          i.remove();
-        } else {
-          // since it failed to delete add it back so it gets retried
-          set.add(resource.getKey(), resource, resource.isUncompressed());
-        }
-      }
-    }
-  }
-
-  protected boolean deleteResource(LocalizedResource resource){
-    try {
-      String fileWithVersion = resource.getFilePathWithVersion();
-      String currentSymlinkName = resource.getCurrentSymlinkPath();
-      String versionFile = resource.getVersionFilePath();
-      File deletePath = new File(fileWithVersion);
-      if (resource.isUncompressed()) {
-        // this doesn't follow symlinks, which is what we want
-        FileUtils.deleteDirectory(deletePath);
-      } else {
-        Files.delete(deletePath.toPath());
-      }
-      Files.delete(new File(currentSymlinkName).toPath());
-      Files.delete(new File(versionFile).toPath());
-      return true;
-    } catch (IOException e) {
-      LOG.warn("Could not delete: {}", resource.getFilePath());
-    }
-    return false;
-  }
-
-  @Override
-  public String toString() {
-    StringBuilder sb = new StringBuilder();
-    sb.append("Cache: ").append(_currentSize).append(", ");
-    sb.append("Deleted: ").append(_delSize);
-    return sb.toString();
-  }
-
-  static class LRUComparator implements Comparator<LocalizedResource> {
-    public int compare(LocalizedResource r1, LocalizedResource r2) {
-      long ret = r1.getLastAccessTime() - r2.getLastAccessTime();
-      if (0 == ret) {
-        return System.identityHashCode(r1) - System.identityHashCode(r2);
-      }
-      return ret > 0 ? 1 : -1;
-    }
-  }
+    public static final Logger LOG = LoggerFactory.getLogger(LocalizedResourceRetentionSet.class);
+    private long delSize;
+    private long currentSize;
+    // targetSize in Bytes
+    private long targetSize;
+    @VisibleForTesting
+    final SortedMap<ComparableResource, CleanableResourceSet> noReferences;
+    private int resourceCount = 0;
+
+    LocalizedResourceRetentionSet(long targetSize) {
+        this(targetSize, new LRUComparator());
+    }
+
+    LocalizedResourceRetentionSet(long targetSize, Comparator<? super ComparableResource> cmp) {
+        this(targetSize, new TreeMap<>(cmp));
+    }
+
+    LocalizedResourceRetentionSet(long targetSize,
+                                  SortedMap<ComparableResource, CleanableResourceSet> retain) {
+        this.noReferences = retain;
+        this.targetSize = targetSize;
+    }
+
+    // for testing
+    protected int getSizeWithNoReferences() {
+        return noReferences.size();
+    }
+
+    protected void addResourcesForSet(Iterator<LocalizedResource> setIter, LocalizedResourceSet set) {
+        CleanableLocalizedResourceSet cleanset = new CleanableLocalizedResourceSet(set);
+        for (Iterator<LocalizedResource> iter = setIter; setIter.hasNext(); ) {
+            LocalizedResource lrsrc = iter.next();
+            currentSize += lrsrc.getSize();
+            resourceCount ++;
+            if (lrsrc.getRefCount() > 0) {
+                // always retain resources in use
+                continue;
+            }
+            noReferences.put(new LocalizedBlobComparableResource(lrsrc), cleanset);
+        }
+    }
+
+    public void addResources(LocalizedResourceSet set) {
+        addResourcesForSet(set.getLocalFilesIterator(), set);
+        addResourcesForSet(set.getLocalArchivesIterator(), set);
+    }
+
+    public void addResources(ConcurrentHashMap<String, LocallyCachedBlob> blobs) {
+        CleanableLocalizedLocallyCachedBlob set = new CleanableLocalizedLocallyCachedBlob(blobs);
+        for (LocallyCachedBlob b: blobs.values()) {
+            currentSize += b.getSizeOnDisk();
+            resourceCount ++;
+            if (b.isUsed()) {
+                // always retain resources in use
+                continue;
+            }
+            LocallyCachedBlobComparableResource cb = new LocallyCachedBlobComparableResource(b);
+            noReferences.put(cb, set);
+        }
+    }
+
+    public void cleanup() {
+        LOG.debug("cleanup target size: {} current size is: {}", targetSize, currentSize);
+        for (Iterator<Map.Entry<ComparableResource, CleanableResourceSet>> i =
+             noReferences.entrySet().iterator();
+             currentSize - delSize > targetSize && i.hasNext();) {
+            Map.Entry<ComparableResource, CleanableResourceSet> rsrc = i.next();
+            ComparableResource resource = rsrc.getKey();
+            CleanableResourceSet set = rsrc.getValue();
+            if (resource != null && set.remove(resource)) {
+                if (set.deleteUnderlyingResource(resource)) {
+                    delSize += resource.getSize();
+                    LOG.info("deleting: {} with size of: {}", resource.getNameForDebug(), resource.getSize());
+                    i.remove();
+                } else {
+                    // since it failed to delete add it back so it gets retried
+                    set.add(resource.getKey(), resource);
+                }
+            }
+        }
+    }
+
+    @VisibleForTesting
+    public boolean deleteResource(CleanableResourceSet set, ComparableResource resource) {
+        return set.deleteUnderlyingResource(resource);
+    }
+
+    public long getCurrentSize() {
+        return currentSize;
+    }
+
+    public int getResourceCount() {
+        return resourceCount;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("Cache: ").append(currentSize).append(", ");
+        sb.append("Deleted: ").append(delSize);
+        return sb.toString();
+    }
+
+    interface ComparableResource {
+        long getLastAccessTime();
+
+        long getSize();
+
+        String getNameForDebug();
+
+        String getKey();
+    }
+
+    interface CleanableResourceSet {
+        boolean remove(ComparableResource resource);
+
+        void add(String key, ComparableResource resource);
+
+        boolean deleteUnderlyingResource(ComparableResource resource);
+    }
+
+    public static class LocallyCachedBlobComparableResource implements ComparableResource {
+        private final LocallyCachedBlob blob;
+
+        public LocallyCachedBlobComparableResource(LocallyCachedBlob blob) {
+            this.blob = blob;
+        }
+
+        @Override
+        public long getLastAccessTime() {
+            return blob.getLastUsed();
+        }
+
+        @Override
+        public long getSize() {
+            return blob.getSizeOnDisk();
+        }
+
+        @Override
+        public String getNameForDebug() {
+            return blob.getKey();
+        }
+
+        @Override
+        public String getKey() {
+            return blob.getKey();
+        }
+
+        @Override
+        public String toString() {
+            return blob.toString();
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (other instanceof LocallyCachedBlobComparableResource) {
+                return blob.equals(((LocallyCachedBlobComparableResource) other).blob);
+            }
+            return false;
+        }
+
+        @Override
+        public int hashCode() {
+            return blob.hashCode();
+        }
+    }
+
+    private static class CleanableLocalizedLocallyCachedBlob implements CleanableResourceSet {
+        private final ConcurrentHashMap<String, LocallyCachedBlob> blobs;
+
+        public CleanableLocalizedLocallyCachedBlob(ConcurrentHashMap<String, LocallyCachedBlob> blobs) {
+            this.blobs = blobs;
+        }
+
+        @Override
+        public boolean remove(ComparableResource resource) {
+            if (!(resource instanceof LocallyCachedBlobComparableResource)) {
+                throw new IllegalStateException(resource + " must be a LocallyCachedBlobComparableResource");
+            }
+            LocallyCachedBlob blob = ((LocallyCachedBlobComparableResource)resource).blob;
+            synchronized (blob) {
+                if (!blob.isUsed()) {
+                    try {
+                        blob.completelyRemove();
+                    } catch (Exception e) {
+                        LOG.warn("Tried to remove {} but failed with", blob, e);
+                    }
+                    blobs.remove(blob.getKey());
+                    return true;
+                }
+                return false;
+            }
+        }
+
+        @Override
+        public void add(String key, ComparableResource resource) {
+            ///NOOP not used
+        }
+
+        @Override
+        public boolean deleteUnderlyingResource(ComparableResource resource) {
+            //NOOP not used
+            return true;
+        }
+    }
+
+    private static class LocalizedBlobComparableResource implements ComparableResource {
+        private final LocalizedResource resource;
+
+        private LocalizedBlobComparableResource(LocalizedResource resource) {
+            this.resource = resource;
+        }
+
+        @Override
+        public long getLastAccessTime() {
+            return resource.getLastAccessTime();
+        }
+
+        @Override
+        public long getSize() {
+            return resource.getSize();
+        }
+
+        @Override
+        public String getNameForDebug() {
+            return resource.getFilePath();
+        }
+
+        @Override
+        public String getKey() {
+            return resource.getKey();
+        }
+
+        @Override
+        public String toString() {
+            return resource.getKey() + " at " + resource.getFilePathWithVersion();
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (other instanceof LocalizedBlobComparableResource) {
+                return resource.equals(((LocalizedBlobComparableResource) other).resource);
+            }
+            return false;
+        }
+
+        @Override
+        public int hashCode() {
+            return resource.hashCode();
+        }
+    }
+
+    private static class CleanableLocalizedResourceSet implements CleanableResourceSet {
+        private final LocalizedResourceSet set;
+
+        public CleanableLocalizedResourceSet(LocalizedResourceSet set) {
+            this.set = set;
+        }
+
+        @Override
+        public boolean remove(ComparableResource resource) {
+            if (!(resource instanceof LocalizedBlobComparableResource)) {
+                throw new IllegalStateException(resource + " must be a LocalizedBlobComparableResource");
+            }
+            return set.remove(((LocalizedBlobComparableResource)resource).resource);
+        }
+
+        @Override
+        public void add(String key, ComparableResource resource) {
+            if (!(resource instanceof LocalizedBlobComparableResource)) {
+                throw new IllegalStateException(resource + " must be a LocalizedBlobComparableResource");
+            }
+            LocalizedResource r = ((LocalizedBlobComparableResource)resource).resource;
+            set.add(key, r, r.isUncompressed());
+        }
+
+        @Override
+        public boolean deleteUnderlyingResource(ComparableResource resource) {
+            if (resource instanceof LocalizedBlobComparableResource) {
+                LocalizedResource lr = ((LocalizedBlobComparableResource) resource).resource;
+                try {
+                    Path fileWithVersion = new File(lr.getFilePathWithVersion()).toPath();
+                    Path currentSymLink = new File(lr.getCurrentSymlinkPath()).toPath();
+                    Path versionFile = new File(lr.getVersionFilePath()).toPath();
+
+                    if (lr.isUncompressed()) {
+                        if (Files.exists(fileWithVersion)) {
+                            // this doesn't follow symlinks, which is what we want
+                            FileUtils.deleteDirectory(fileWithVersion.toFile());
+                        }
+                    } else {
+                        Files.deleteIfExists(fileWithVersion);
+                    }
+                    Files.deleteIfExists(currentSymLink);
+                    Files.deleteIfExists(versionFile);
+                    return true;
+                } catch (IOException e) {
+                    LOG.warn("Could not delete: {}", resource.getNameForDebug(), e);
+                }
+                return false;
+            }  else {
+                throw new IllegalArgumentException("Don't know how to handle a " + resource.getClass());
+            }
+        }
+    }
+
+    static class LRUComparator implements Comparator<ComparableResource> {
+        public int compare(ComparableResource r1, ComparableResource r2) {
+            long ret = r1.getLastAccessTime() - r2.getLastAccessTime();
+            if (0 == ret) {
+                return System.identityHashCode(r1) - System.identityHashCode(r2);
+            }
+            return ret > 0 ? 1 : -1;
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java
new file mode 100644
index 0000000..c09108d
--- /dev/null
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.localizer;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.KeyNotFoundException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents a blob that is cached locally on disk by the supervisor.
+ */
+public abstract class LocallyCachedBlob {
+    private static final Logger LOG = LoggerFactory.getLogger(LocallyCachedBlob.class);
+    public static final long NOT_DOWNLOADED_VERSION = -1;
+    // A callback that does nothing.
+    private static final BlobChangingCallback NOOP_CB = (assignment, port, resource, go) -> {};
+
+    private long lastUsed = System.currentTimeMillis();
+    private final Map<PortAndAssignment, BlobChangingCallback> references = new HashMap<>();
+    private final String blobDescription;
+    private final String blobKey;
+    private CompletableFuture<Void> doneUpdating = null;
+
+    /**
+     * Create a new LocallyCachedBlob.
+     * @param blobDescription a description of the blob this represents.  Typically it should at least be the blob key, but ideally also
+     * include if it is an archive or not, what user or topology it is for, or if it is a storm.jar etc.
+     */
+    protected LocallyCachedBlob(String blobDescription, String blobKey) {
+        this.blobDescription = blobDescription;
+        this.blobKey = blobKey;
+    }
+
+    /**
+     * Get the version of the blob cached locally.  If the version is unknown or it has not been downloaded NOT_DOWNLOADED_VERSION
+     * should be returned.
+     * PRECONDITION: this can only be called with a lock on this instance held.
+     */
+    public abstract long getLocalVersion();
+
+    /**
+     * Get the version of the blob in the blob store.
+     * PRECONDITION: this can only be called with a lock on this instance held.
+     */
+    public abstract long getRemoteVersion(ClientBlobStore store) throws KeyNotFoundException, AuthorizationException;
+
+    /**
+     * Download the latest version to a temp location. This may also include unzipping some or all of the data to a temp location.
+     * PRECONDITION: this can only be called with a lock on this instance held.
+     * @param store the store to us to download the data.
+     * @return the version that was downloaded.
+     */
+    public abstract long downloadToTempLocation(ClientBlobStore store) throws IOException, KeyNotFoundException, AuthorizationException;
+
+    /**
+     * Commit the new version and make it available for the end user.
+     * PRECONDITION: uncompressToTempLocationIfNeeded will have been called.
+     * PRECONDITION: this can only be called with a lock on this instance held.
+     * @param version the version of the blob to commit.
+     */
+    public abstract void commitNewVersion(long version) throws IOException;
+
+    /**
+     * Clean up any temporary files.  This will be called after updating a blob, either successfully or if an error has occured.
+     * The goal is to find any files that may be left over and remove them so space is not leaked.
+     * PRECONDITION: this can only be called with a lock on this instance held.
+     */
+    public abstract void cleanupOrphanedData() throws IOException;
+
+    /**
+     * Completely remove anything that is cached locally for this blob and all tracking files also stored for it.
+     * This will be called after the blob was determined to no longer be needed in the cache.
+     * PRECONDITION: this can only be called with a lock on this instance held.
+     */
+    public abstract void completelyRemove() throws IOException;
+
+    /**
+     * Get the amount of disk space that is used by this blob.  If the blob is uncompressed it should be the sum of the space used by all
+     * of the uncompressed files.  In general this will not be called with any locks held so it is a good idea to cache it and updated it
+     * when committing a new version.
+     */
+    public abstract long getSizeOnDisk();
+
+    /**
+     * Updates the last updated time.  This should be called when references are added or removed.
+     */
+    private synchronized void touch() {
+        lastUsed = System.currentTimeMillis();
+    }
+
+    /**
+     * Get the last time that this used for LRU calculations.
+     */
+    public synchronized long getLastUsed() {
+        return lastUsed;
+    }
+
+    /**
+     * Return true if this blob is actively being used, else false (meaning it can be deleted, but might not be).
+     */
+    public synchronized boolean isUsed() {
+        return !references.isEmpty();
+    }
+
+    /**
+     * Get the size of p in bytes.
+     * @param p the path to read.
+     * @return the size of p in bytes.
+     */
+    protected long getSizeOnDisk(Path p) throws IOException {
+        if (!Files.exists(p)) {
+            return 0;
+        } else if (Files.isRegularFile(p)) {
+            return Files.size(p);
+        } else {
+            //We will not follow sym links
+            return Files.walk(p)
+                .filter((subp) -> Files.isRegularFile(subp, LinkOption.NOFOLLOW_LINKS))
+                .mapToLong((subp) -> {
+                    try {
+                        return Files.size(subp);
+                    } catch (IOException e) {
+                        LOG.warn("Could not get the size of ");
+                    }
+                    return 0;
+                }).sum();
+        }
+    }
+
+    /**
+     * Mark that a given port and assignemnt are using this.
+     * @param pna the slot and assignment that are using this blob.
+     * @param cb an optional callback indicating that they want to know/synchronize when a blob is updated.
+     */
+    public void addReference(final PortAndAssignment pna, BlobChangingCallback cb) {
+        if (cb == null) {
+            cb = NOOP_CB;
+        }
+        if (references.put(pna, cb) != null) {
+            LOG.warn("{} already has a reservation for {}", pna, blobDescription);
+        }
+    }
+
+    /**
+     * Removes a reservation for this blob from a given slot and assignemnt.
+     * @param pna the slot + assignment that no longer needs this blob.
+     */
+    public void removeReference(final PortAndAssignment pna) {
+        if (references.remove(pna) == null) {
+            LOG.warn("{} had no reservation for {}", pna, blobDescription);
+        }
+    }
+
+    /**
+     * Inform all of the callbacks that a change is going to happen and then wait for
+     * them to all get back that it is OK to make that change.
+     */
+    public synchronized void informAllOfChangeAndWaitForConsensus() {
+        CountDownLatch cdl = new CountDownLatch(references.size());
+        doneUpdating = new CompletableFuture<>();
+        for (Map.Entry<PortAndAssignment, BlobChangingCallback> entry : references.entrySet()) {
+            GoodToGo gtg = new GoodToGo(cdl, doneUpdating);
+            try {
+                PortAndAssignment pna = entry.getKey();
+                BlobChangingCallback cb = entry.getValue();
+                //TODO we probably want to not use this, or make it just return something that has less power to modify things
+                cb.blobChanging(pna.getAssignment(), pna.getPort(), this, gtg);
+            } finally {
+                gtg.countDownIfLatchWasNotGotten();
+            }
+        }
+        try {
+            cdl.await(3, TimeUnit.MINUTES);
+        } catch (InterruptedException e) {
+            //TODO need to think about error handling here in general.
+        }
+    }
+
+    /**
+     * Inform all of the callbacks that the change to the blob is complete.
+     */
+    public synchronized void informAllChangeComplete() {
+        doneUpdating.complete(null);
+    }
+
+    /**
+     * Get the key for this blob.
+     */
+    public String getKey() {
+        return blobKey;
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java
new file mode 100644
index 0000000..35371b5
--- /dev/null
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java
@@ -0,0 +1,364 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.localizer;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.JarURLConnection;
+import java.net.URL;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Enumeration;
+import java.util.Map;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.blobstore.InputStreamWithMeta;
+import org.apache.storm.daemon.supervisor.AdvancedFSOps;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.KeyNotFoundException;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ServerConfigUtils;
+import org.apache.storm.utils.ServerUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A locally cached blob for the topology.  storm.jar, stormcode.ser, or stormconf.ser.
+ * The version number of the blob's file will be stored in `${basename}.version`
+ */
+public class LocallyCachedTopologyBlob extends LocallyCachedBlob {
+    private static final Logger LOG = LoggerFactory.getLogger(LocallyCachedTopologyBlob.class);
+    public static final long LOCAL_MODE_JAR_VERSION = 1;
+
+    private static String resourcesJar() throws IOException {
+        String path = ServerUtils.currentClasspath();
+        if (path == null) {
+            return null;
+        }
+
+        for (String jpath : path.split(File.pathSeparator)) {
+            if (jpath.endsWith(".jar")) {
+                if (ServerUtils.zipDoesContainDir(jpath, ServerConfigUtils.RESOURCES_SUBDIR)) {
+                    return jpath;
+                }
+            }
+        }
+        return null;
+    }
+
+    public enum TopologyBlobType {
+        TOPO_JAR("stormjar.jar", "-stormjar.jar", "resources"),
+        TOPO_CODE("stormcode.ser", "-stormcode.ser", null),
+        TOPO_CONF("stormconf.ser", "-stormconf.ser", null);
+
+        private final String fileName;
+        private final String keySuffix;
+        private final String extractionDir;
+
+        TopologyBlobType(String fileName, String keySuffix, String extractionDir) {
+            this.fileName = fileName;
+            this.keySuffix = keySuffix;
+            this.extractionDir = extractionDir;
+        }
+
+        public String getFileName() {
+            return fileName;
+        }
+
+        public String getTempFileName(long version) {
+            return fileName + "." + version;
+        }
+
+        public String getVersionFileName() {
+            return fileName + ".version";
+        }
+
+        public String getKey(String topologyId) {
+            return topologyId + keySuffix;
+        }
+
+        public boolean needsExtraction() {
+            return extractionDir != null;
+        }
+
+        public String getExtractionDir() {
+            return extractionDir;
+        }
+
+        public String getTempExtractionDir(long version) {
+            return extractionDir + "." + version;
+        }
+    };
+
+    private final TopologyBlobType type;
+    private final String topologyId;
+    private final boolean isLocalMode;
+    private final Path topologyBasicBlobsRootDir;
+    private final AdvancedFSOps fsOps;
+    private final Map<String, Object> conf;
+    private volatile long version = NOT_DOWNLOADED_VERSION;
+    private volatile long size = 0;
+
+    /**
+     * Create a new LocallyCachedBlob.
+     *
+     * @param topologyId the ID of the topology.
+     * @param type the type of the blob.
+     */
+    protected LocallyCachedTopologyBlob(final String topologyId, final boolean isLocalMode, final Map<String, Object> conf,
+                                        final AdvancedFSOps fsOps, final TopologyBlobType type) throws IOException {
+        super(topologyId + " " + type.getFileName(), type.getKey(topologyId));
+        this.topologyId = topologyId;
+        this.type = type;
+        this.isLocalMode = isLocalMode;
+        this.fsOps = fsOps;
+        this.conf = conf;
+        topologyBasicBlobsRootDir = Paths.get(ConfigUtils.supervisorStormDistRoot(conf, topologyId));
+        readVersion();
+        updateSizeOnDisk();
+    }
+
+    private void updateSizeOnDisk() throws IOException {
+        long total = getSizeOnDisk(topologyBasicBlobsRootDir.resolve(type.getFileName()));
+        if (type.needsExtraction()) {
+            total += getSizeOnDisk(topologyBasicBlobsRootDir.resolve(type.getExtractionDir()));
+        }
+        size = total;
+    }
+
+    private void readVersion() throws IOException {
+        Path versionFile = topologyBasicBlobsRootDir.resolve(type.getVersionFileName());
+        if (!fsOps.fileExists(versionFile)) {
+            version = NOT_DOWNLOADED_VERSION;
+        } else {
+            String ver = FileUtils.readFileToString(versionFile.toFile(), "UTF8").trim();
+            version = Long.parseLong(ver);
+        }
+    }
+
+    @Override
+    public long getLocalVersion() {
+        LOG.debug("LOCAL VERSION {}/{} is {}", type, topologyId, version);
+        return version;
+    }
+
+    @Override
+    public long getRemoteVersion(ClientBlobStore store) throws KeyNotFoundException, AuthorizationException {
+        if (isLocalMode && type == TopologyBlobType.TOPO_JAR) {
+            LOG.debug("REMOTE VERSION LOCAL JAR {}", LOCAL_MODE_JAR_VERSION);
+            return LOCAL_MODE_JAR_VERSION;
+        }
+        return store.getBlobMeta(type.getKey(topologyId)).get_version();
+    }
+
+    @Override
+    public long downloadToTempLocation(ClientBlobStore store)
+        throws IOException, KeyNotFoundException, AuthorizationException {
+        if (isLocalMode && type == TopologyBlobType.TOPO_JAR) {
+            LOG.debug("DOWNLOADING LOCAL JAR to TEMP LOCATION... {}", topologyId);
+            //This is a special case where the jar was not uploaded so we will not download it (it is already on the classpath)
+            ClassLoader classloader = Thread.currentThread().getContextClassLoader();
+            String resourcesJar = resourcesJar();
+            URL url = classloader.getResource(ServerConfigUtils.RESOURCES_SUBDIR);
+            Path extractionDest = topologyBasicBlobsRootDir.resolve(type.getTempExtractionDir(LOCAL_MODE_JAR_VERSION));
+            if (resourcesJar != null) {
+                LOG.info("Extracting resources from jar at {} to {}", resourcesJar, extractionDest);
+                extractDirFromJar(resourcesJar, ServerConfigUtils.RESOURCES_SUBDIR, extractionDest);
+            } else if (url != null) {
+                LOG.info("Copying resources at {} to {}", url, extractionDest);
+                if ("jar".equals(url.getProtocol())) {
+                    JarURLConnection urlConnection = (JarURLConnection) url.openConnection();
+                    extractDirFromJar(urlConnection.getJarFileURL().getFile(), ServerConfigUtils.RESOURCES_SUBDIR, extractionDest);
+                } else {
+                    fsOps.copyDirectory(new File(url.getFile()), extractionDest.toFile());
+                }
+            }
+            return LOCAL_MODE_JAR_VERSION;
+        }
+
+        long newVersion;
+        Path tmpLocation;
+        String key = type.getKey(topologyId);
+        try (InputStreamWithMeta in = store.getBlob(key)) {
+            newVersion = in.getVersion();
+            long expectedSize = in.getFileLength();
+            if (newVersion == version) {
+                throw new RuntimeException("The version did not change, but we tried to download it. " + version + " " + key);
+            }
+            tmpLocation = topologyBasicBlobsRootDir.resolve(type.getTempFileName(newVersion));
+            long totalRead = 0;
+            //Make sure the parent directory is there and ready to go
+            fsOps.forceMkdir(tmpLocation.getParent());
+            try (OutputStream outStream = fsOps.getOutputStream(tmpLocation.toFile())) {
+                byte [] buffer = new byte[4096];
+                int read = 0;
+                while ((read = in.read(buffer)) > 0) {
+                    outStream.write(buffer, 0, read);
+                    totalRead += read;
+                }
+            }
+            if (totalRead != expectedSize) {
+                throw new IOException("We expected to download " + expectedSize + " bytes but found we got " + totalRead);
+            }
+        }
+
+        if (type.needsExtraction()) {
+            Path extractionDest = topologyBasicBlobsRootDir.resolve(type.getTempExtractionDir(newVersion));
+            extractDirFromJar(tmpLocation.toAbsolutePath().toString(), ServerConfigUtils.RESOURCES_SUBDIR,
+                extractionDest);
+        }
+        return newVersion;
+    }
+
+    protected void extractDirFromJar(String jarpath, String dir, Path dest) throws IOException {
+        LOG.debug("EXTRACTING {} from {} and placing it at {}", dir, jarpath, dest);
+        try (JarFile jarFile = new JarFile(jarpath)) {
+            String toRemove = dir + '/';
+            Enumeration<JarEntry> jarEnums = jarFile.entries();
+            while (jarEnums.hasMoreElements()) {
+                JarEntry entry = jarEnums.nextElement();
+                String name = entry.getName();
+                if (!entry.isDirectory() && name.startsWith(toRemove)) {
+                    String shortenedName = name.replace(toRemove, "");
+                    Path aFile = dest.resolve(shortenedName);
+                    LOG.debug("EXTRACTING {} SHORTENED to {} into {}", name, shortenedName, aFile);
+                    fsOps.forceMkdir(aFile.getParent());
+                    try (FileOutputStream out = new FileOutputStream(aFile.toFile());
+                         InputStream in = jarFile.getInputStream(entry)) {
+                        IOUtils.copy(in, out);
+                    }
+                }
+            }
+        }
+    }
+
+    @Override
+    public void commitNewVersion(long newVersion) throws IOException {
+        //This is not atomic (so if something bad happens in the middle we need to be able to recover
+        Path tempLoc = topologyBasicBlobsRootDir.resolve(type.getTempFileName(newVersion));
+        Path dest = topologyBasicBlobsRootDir.resolve(type.getFileName());
+        Path versionFile = topologyBasicBlobsRootDir.resolve(type.getVersionFileName());
+
+        LOG.debug("Removing version file {} to force download on failure", versionFile);
+        fsOps.deleteIfExists(versionFile.toFile()); //So if we fail we are forced to try again
+        LOG.debug("Removing destination file {} in preparation for move", dest);
+        fsOps.deleteIfExists(dest.toFile());
+        if (type.needsExtraction()) {
+            Path extractionTemp = topologyBasicBlobsRootDir.resolve(type.getTempExtractionDir(newVersion));
+            Path extractionDest = topologyBasicBlobsRootDir.resolve(type.getExtractionDir());
+            LOG.debug("Removing extraction dest {} in preparation for extraction", extractionDest);
+            fsOps.deleteIfExists(extractionDest.toFile());
+            if (fsOps.fileExists(extractionTemp)) {
+                fsOps.moveDirectoryPreferAtomic(extractionTemp.toFile(), extractionDest.toFile());
+            }
+        }
+        if (!(isLocalMode && type == TopologyBlobType.TOPO_JAR)) {
+            //Don't try to move the JAR file in local mode, it does not exist because it was not uploaded
+            Files.move(tempLoc, dest);
+        }
+        LOG.debug("Writing out version file {} with version {}", versionFile, newVersion);
+        FileUtils.write(versionFile.toFile(), Long.toString(newVersion), "UTF8");
+        this.version = newVersion;
+        updateSizeOnDisk();
+        LOG.debug("New version of {} - {} committed {}", topologyId, type, newVersion);
+    }
+
+    @Override
+    public void cleanupOrphanedData() throws IOException {
+        cleanUpTemp(type.getFileName());
+        if (type.needsExtraction()) {
+            cleanUpTemp(type.getExtractionDir());
+        }
+    }
+
+    private static final Pattern EXTRACT_BASE_NAME_AND_VERSION = Pattern.compile("^(.*)\\.([0-9]+)$");
+
+    private void cleanUpTemp(String baseName) throws IOException {
+        LOG.debug("Cleaning up temporary data in {}", topologyBasicBlobsRootDir);
+        try (DirectoryStream<Path> children = fsOps.newDirectoryStream(topologyBasicBlobsRootDir,
+            (p) -> {
+                String fileName = p.getFileName().toString();
+                Matcher m = EXTRACT_BASE_NAME_AND_VERSION.matcher(fileName);
+                return m.matches() && baseName.equals(m.group(1));
+            })) {
+            //children is only ever null if topologyBasicBlobsRootDir does not exist.  This happens during unit tests
+            // And because a non-existant directory is by definition clean we are ignoring it.
+            if (children != null) {
+                for (Path p : children) {
+                    LOG.debug("Cleaning up {}", p);
+                    fsOps.deleteIfExists(p.toFile());
+                }
+            }
+        }
+    }
+
+    @Override
+    public void completelyRemove() throws IOException {
+        removeAll(type.getFileName());
+        if (type.needsExtraction()) {
+            removeAll(type.getExtractionDir());
+        }
+    }
+
+    private void removeAll(String baseName) throws IOException {
+        try (DirectoryStream<Path> children = fsOps.newDirectoryStream(topologyBasicBlobsRootDir)) {
+            for (Path p : children) {
+                String fileName = p.getFileName().toString();
+                if (fileName.startsWith(baseName)) {
+                    fsOps.deleteIfExists(p.toFile());
+                }
+            }
+        }
+    }
+
+    @Override
+    public long getSizeOnDisk() {
+        return size;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other instanceof LocallyCachedTopologyBlob) {
+            LocallyCachedTopologyBlob o = (LocallyCachedTopologyBlob)other;
+            return topologyId.equals(o.topologyId) && type == o.type && topologyBasicBlobsRootDir.equals(o.topologyBasicBlobsRootDir);
+        }
+        return false;
+    }
+
+    @Override
+    public int hashCode() {
+        return topologyId.hashCode() + type.hashCode() + topologyBasicBlobsRootDir.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return "LOCAL TOPO BLOB " + type + " " + topologyId;
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java b/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java
new file mode 100644
index 0000000..081c811
--- /dev/null
+++ b/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.localizer;
+
+import org.apache.storm.generated.LocalAssignment;
+
+/**
+ * A Port and a LocalAssignment used to reference count Resources
+ */
+class PortAndAssignment {
+    private final int port;
+    private final LocalAssignment assignment;
+
+    public PortAndAssignment(int port, LocalAssignment assignment) {
+        this.port = port;
+        this.assignment = assignment;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (!(other instanceof PortAndAssignment)) {
+            return false;
+        }
+        PortAndAssignment pna = (PortAndAssignment) other;
+        return pna.port == port && assignment.equals(pna.assignment);
+    }
+
+    public String getToplogyId() {
+        return assignment.get_topology_id();
+    }
+
+    @Override
+    public int hashCode() {
+        return (17 * port) + assignment.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return "{" + port + " " + assignment + "}";
+    }
+
+    /**
+     * Return the port associated with this.
+     */
+    public int getPort() {
+        return port;
+    }
+
+    /**
+     * return the assigment for this.
+     */
+    public LocalAssignment getAssignment() {
+        return assignment;
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
index a50dced..ab8eb14 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
@@ -24,6 +24,7 @@ import java.util.Map;
 
 import org.apache.storm.Config;
 import org.apache.storm.generated.Bolt;
+import org.apache.storm.generated.ComponentCommon;
 import org.apache.storm.generated.SpoutSpec;
 import org.apache.storm.generated.StormTopology;
 import org.apache.storm.utils.ObjectReader;
@@ -63,6 +64,71 @@ public class ResourceUtils {
         return spoutResources;
     }
 
+    public static void updateStormTopologyResources(StormTopology topology, Map<String, Map<String, Double>> resourceUpdatesMap) {
+        Map<String, Map<String, Double>> componentsUpdated = new HashMap<>();
+        if (topology.get_spouts() != null) {
+            for (Map.Entry<String, SpoutSpec> spout : topology.get_spouts().entrySet()) {
+                SpoutSpec spoutSpec = spout.getValue();
+                String spoutName = spout.getKey();
+
+                if (resourceUpdatesMap.containsKey(spoutName)) {
+                    ComponentCommon spoutCommon = spoutSpec.get_common();
+                    Map<String, Double> resourcesUpdate = resourceUpdatesMap.get(spoutName);
+                    String newJsonConf = getJsonWithUpdatedResources(spoutCommon.get_json_conf(), resourcesUpdate);
+                    spoutCommon.set_json_conf(newJsonConf);
+                    componentsUpdated.put(spoutName, resourcesUpdate);
+                }
+            }
+        }
+
+        if (topology.get_bolts() != null) {
+            for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) {
+                Bolt boltObj = bolt.getValue();
+                String boltName = bolt.getKey();
+
+                if(resourceUpdatesMap.containsKey(boltName)) {
+                    ComponentCommon boltCommon = boltObj.get_common();
+                    Map<String, Double> resourcesUpdate = resourceUpdatesMap.get(boltName);
+                    String newJsonConf = getJsonWithUpdatedResources(boltCommon.get_json_conf(), resourceUpdatesMap.get(boltName));
+                    boltCommon.set_json_conf(newJsonConf);
+                    componentsUpdated.put(boltName, resourcesUpdate);
+                }
+            }
+        }
+        LOG.info("Component resources updated: {}", componentsUpdated);
+        Map<String, Map<String, Double>> notUpdated = new HashMap<String, Map<String, Double>>();
+        for (String component : resourceUpdatesMap.keySet()) {
+            if (!componentsUpdated.containsKey(component)) {
+                notUpdated.put(component, resourceUpdatesMap.get(component));
+            }
+        }
+        LOG.info("Component resource updates ignored: {}", notUpdated);
+    }
+
+    public static String getJsonWithUpdatedResources(String jsonConf, Map<String, Double> resourceUpdates) {
+        try {
+            JSONParser parser = new JSONParser();
+            Object obj = parser.parse(jsonConf);
+            JSONObject jsonObject = (JSONObject) obj;
+
+            if (resourceUpdates.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)) {
+                Double topoMemOnHeap = resourceUpdates.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB);
+                jsonObject.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, topoMemOnHeap);
+            }
+            if (resourceUpdates.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)) {
+                Double topoMemOffHeap = resourceUpdates.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB);
+                jsonObject.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, topoMemOffHeap);
+            }
+            if (resourceUpdates.containsKey(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)) {
+                Double topoCPU = resourceUpdates.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT);
+                jsonObject.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, topoCPU);
+            }
+            return jsonObject.toJSONString();
+        } catch (ParseException ex) {
+            throw new RuntimeException("Failed to parse component resources with json: " +  jsonConf);
+        }
+    }
+
     public static void checkIntialization(Map<String, Double> topologyResources, String com,
                                           Map<String, Object> topologyConf) {
         checkInitMem(topologyResources, com, topologyConf);

http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java b/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
index 9f2b95b..6a4454a 100644
--- a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
@@ -18,6 +18,7 @@
 
 package org.apache.storm.utils;
 
+import javax.security.auth.Subject;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.exec.CommandLine;
@@ -32,6 +33,7 @@ import org.apache.storm.blobstore.BlobStoreAclHandler;
 import org.apache.storm.blobstore.ClientBlobStore;
 import org.apache.storm.blobstore.InputStreamWithMeta;
 import org.apache.storm.blobstore.LocalFsBlobStore;
+import org.apache.storm.blobstore.LocalModeClientBlobStore;
 import org.apache.storm.daemon.StormCommon;
 import org.apache.storm.generated.AccessControl;
 import org.apache.storm.generated.AccessControlType;
@@ -43,6 +45,7 @@ import org.apache.storm.generated.SettableBlobMeta;
 import org.apache.storm.generated.StormTopology;
 import org.apache.storm.nimbus.NimbusInfo;
 import org.apache.storm.scheduler.resource.ResourceUtils;
+import org.apache.storm.security.auth.SingleUserPrincipal;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -212,8 +215,13 @@ public class ServerUtils {
     }
 
     public static ClientBlobStore getClientBlobStoreForSupervisor(Map<String, Object> conf) {
-        ClientBlobStore store = (ClientBlobStore) ReflectionUtils.newInstance(
+        ClientBlobStore store;
+        if (ConfigUtils.isLocalMode(conf)) {
+            store = new LocalModeClientBlobStore(getNimbusBlobStore(conf, null));
+        } else {
+            store = (ClientBlobStore) ReflectionUtils.newInstance(
                 (String) conf.get(DaemonConfig.SUPERVISOR_BLOBSTORE));
+        }
         store.prepare(conf);
         return store;
     }
@@ -767,4 +775,11 @@ public class ServerUtils {
         }
         return ret;
     }
+
+    public static Subject principalNameToSubject(String name) {
+        SingleUserPrincipal principal = new SingleUserPrincipal(name);
+        Subject sub = new Subject();
+        sub.getPrincipals().add(principal);
+        return sub;
+    }
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/test/java/org/apache/storm/TestRebalance.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/TestRebalance.java b/storm-server/src/test/java/org/apache/storm/TestRebalance.java
new file mode 100644
index 0000000..bd72b3a
--- /dev/null
+++ b/storm-server/src/test/java/org/apache/storm/TestRebalance.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm;
+
+import org.apache.storm.generated.ClusterSummary;
+import org.apache.storm.generated.RebalanceOptions;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.generated.TopologySummary;
+import org.apache.storm.scheduler.resource.ResourceAwareScheduler;
+import org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler;
+import org.apache.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy;
+import org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy;
+import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy;
+import org.apache.storm.topology.BoltDeclarer;
+import org.apache.storm.topology.SpoutDeclarer;
+import org.apache.storm.topology.TopologyBuilder;
+import org.apache.storm.utils.Utils;
+import org.apache.thrift.TException;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestRebalance {
+
+    static final int SLEEP_TIME_BETWEEN_RETRY = 1000;
+
+    private static final Logger LOG = LoggerFactory.getLogger(TestRebalance.class);
+
+    @Test
+    public void testRebalanceTopologyResourcesAndConfigs()
+        throws Exception {
+
+        LOG.info("Starting local cluster...");
+
+        Config conf = new Config();
+        conf.put(DaemonConfig.STORM_SCHEDULER, ResourceAwareScheduler.class.getName());
+        conf.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY, DefaultEvictionStrategy.class.getName());
+        conf.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, DefaultSchedulingPriorityStrategy.class.getName());
+        conf.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, DefaultResourceAwareStrategy.class.getName());
+        conf.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 10.0);
+        conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 10.0);
+        conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 100.0);
+        conf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
+
+        try (ILocalCluster cluster = new LocalCluster.Builder().withDaemonConf(conf).build()) {
+
+            TopologyBuilder builder = new TopologyBuilder();
+            SpoutDeclarer s1 = builder.setSpout("spout-1", new TestUtilsForResourceAwareScheduler.TestSpout(),
+                2);
+            BoltDeclarer b1 = builder.setBolt("bolt-1", new TestUtilsForResourceAwareScheduler.TestBolt(),
+                2).shuffleGrouping("spout-1");
+            BoltDeclarer b2 = builder.setBolt("bolt-2", new TestUtilsForResourceAwareScheduler.TestBolt(),
+                2).shuffleGrouping("bolt-1");
+
+            StormTopology stormTopology = builder.createTopology();
+
+            LOG.info("submitting topologies...");
+            String topoName = "topo1";
+            cluster.submitTopology(topoName, new HashMap<>(), stormTopology);
+
+            waitTopologyScheduled(topoName, cluster, 20);
+
+            RebalanceOptions opts = new RebalanceOptions();
+
+            Map<String, Map<String, Double>> resources = new HashMap<String, Map<String, Double>>();
+            resources.put("spout-1", new HashMap<String, Double>());
+            resources.get("spout-1").put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 120.0);
+            resources.get("spout-1").put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 25.0);
+
+            opts.set_topology_resources_overrides(resources);
+            opts.set_wait_secs(0);
+
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 768.0);
+
+            opts.set_topology_conf_overrides(jsonObject.toJSONString());
+
+            LOG.info("rebalancing...");
+            cluster.rebalance("topo1", opts);
+
+            waitTopologyScheduled(topoName, cluster, 10);
+
+            String confRaw = cluster.getTopologyConf(topoNameToId(topoName, cluster));
+
+            JSONParser parser = new JSONParser();
+
+            JSONObject readConf = (JSONObject) parser.parse(confRaw);
+            assertEquals("updated conf correct", 768.0, (double) readConf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB), 0.001);
+
+            StormTopology readStormTopology = cluster.getTopology(topoNameToId(topoName, cluster));
+            String componentConfRaw = readStormTopology.get_spouts().get("spout-1").get_common().get_json_conf();
+
+            JSONObject readTopologyConf = (JSONObject) parser.parse(componentConfRaw);
+
+            assertEquals("Updated CPU correct", 25.0, (double) readTopologyConf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT), 0.001);
+            assertEquals("Updated Memory correct", 120.0, (double) readTopologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), 0.001);
+        }
+    }
+
+    public void waitTopologyScheduled(String topoName, ILocalCluster cluster, int retryAttempts) throws TException {
+        for (int i = 0; i < retryAttempts; i++) {
+            if (checkTopologyScheduled(topoName, cluster)) {
+                //sleep to prevent race conditions
+                Utils.sleep(SLEEP_TIME_BETWEEN_RETRY);
+                return;
+            }
+            Utils.sleep(SLEEP_TIME_BETWEEN_RETRY);
+        }
+        throw new RuntimeException("Error: Wait for topology " + topoName + " to be ACTIVE has timed out!");
+    }
+
+    public boolean checkTopologyScheduled(String topoName, ILocalCluster cluster) throws TException {
+        if (checkTopologyUp(topoName, cluster)) {
+            ClusterSummary sum = cluster.getClusterInfo();
+            for (TopologySummary topoSum : sum.get_topologies()) {
+                if (topoSum.get_name().equals(topoName)) {
+                    String status = topoSum.get_status();
+                    String sched_status = topoSum.get_sched_status();
+                    if (status.equals("ACTIVE") && (sched_status != null && !sched_status.equals(""))) {
+                        return true;
+                    }
+                }
+            }
+        }
+        return false;
+    }
+
+    public boolean checkTopologyUp(String topoName, ILocalCluster cluster) throws TException {
+        ClusterSummary sum = cluster.getClusterInfo();
+
+        for (TopologySummary topoSum : sum.get_topologies()) {
+            if (topoSum.get_name().equals(topoName)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static String topoNameToId(String topoName, ILocalCluster cluster) throws TException {
+        for (TopologySummary topoSum : cluster.getClusterInfo().get_topologies()) {
+            if (topoSum.get_name().equals(topoName)) {
+                return topoSum.get_id();
+            }
+        }
+        return null;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
index 60b628e..927225d 100644
--- a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
+++ b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
@@ -26,6 +26,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.storm.daemon.supervisor.Slot.StaticState;
@@ -40,13 +41,20 @@ import org.apache.storm.generated.ProfileAction;
 import org.apache.storm.generated.ProfileRequest;
 import org.apache.storm.generated.WorkerResources;
 import org.apache.storm.localizer.AsyncLocalizer;
+import org.apache.storm.localizer.BlobChangingCallback;
+import org.apache.storm.localizer.GoodToGo;
+import org.apache.storm.localizer.LocallyCachedBlob;
 import org.apache.storm.scheduler.ISupervisor;
 import org.apache.storm.utils.LocalState;
 import org.apache.storm.utils.Time;
 import org.apache.storm.utils.Time.SimulatedTime;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class SlotTest {
+    private static final Logger LOG = LoggerFactory.getLogger(SlotTest.class);
+
     static WorkerResources mkWorkerResources(Double cpu, Double mem_on_heap, Double mem_off_heap) {
         WorkerResources resources = new WorkerResources();
         if (cpu != null) {
@@ -111,16 +119,36 @@ public class SlotTest {
         assertFalse(Slot.equivalent(null, b));
         assertFalse(Slot.equivalent(a, b));
     }
-    
+
+    @Test
+    public void testForSameTopology() {
+        LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 100.0, 100.0));
+        LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 200.0, 100.0));
+        LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1,2,3,4,5,6), mkWorkerResources(100.0, 100.0, 100.0));
+        LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6,5,4,3,2,1), mkWorkerResources(100.0, 100.0, 100.0));
+
+        assertTrue(Slot.forSameTopology(null, null));
+        assertTrue(Slot.forSameTopology(a, a));
+        assertTrue(Slot.forSameTopology(a, aResized));
+        assertTrue(Slot.forSameTopology(aResized, a));
+        assertTrue(Slot.forSameTopology(b, bReordered));
+        assertTrue(Slot.forSameTopology(bReordered, b));
+
+        assertFalse(Slot.forSameTopology(a, null));
+        assertFalse(Slot.forSameTopology(null, b));
+        assertFalse(Slot.forSameTopology(a, b));
+    }
+
     @Test
     public void testEmptyToEmpty() throws Exception {
         try (SimulatedTime t = new SimulatedTime(1010)){
             AsyncLocalizer localizer = mock(AsyncLocalizer.class);
             LocalState state = mock(LocalState.class);
+            BlobChangingCallback cb = mock(BlobChangingCallback.class);
             ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
             ISupervisor iSuper = mock(ISupervisor.class);
             StaticState staticState = new StaticState(localizer, 1000, 1000, 1000, 1000,
-                    containerLauncher, "localhost", 8080, iSuper, state);
+                    containerLauncher, "localhost", 8080, iSuper, state, cb);
             DynamicState dynamicState = new DynamicState(null, null, null);
             DynamicState nextState = Slot.handleEmpty(dynamicState, staticState);
             assertEquals(MachineState.EMPTY, nextState.state);
@@ -136,39 +164,28 @@ public class SlotTest {
             List<ExecutorInfo> execList =  mkExecutorInfoList(1,2,3,4,5);
             LocalAssignment newAssignment = 
                     mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
-            
+
             AsyncLocalizer localizer = mock(AsyncLocalizer.class);
+            BlobChangingCallback cb = mock(BlobChangingCallback.class);
             Container container = mock(Container.class);
             LocalState state = mock(LocalState.class);
             ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
             when(containerLauncher.launchContainer(port, newAssignment, state)).thenReturn(container);
             LSWorkerHeartbeat hb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs());
             when(container.readHeartbeat()).thenReturn(hb, hb);
-            
-            @SuppressWarnings("unchecked")
-            CompletableFuture<Void> baseFuture = mock(CompletableFuture.class);
-            when(localizer.requestDownloadBaseTopologyBlobs(newAssignment, port)).thenReturn(baseFuture);
-            
+
             @SuppressWarnings("unchecked")
             CompletableFuture<Void> blobFuture = mock(CompletableFuture.class);
-            when(localizer.requestDownloadTopologyBlobs(newAssignment, port)).thenReturn(blobFuture);
+            when(localizer.requestDownloadTopologyBlobs(newAssignment, port, cb)).thenReturn(blobFuture);
             
             ISupervisor iSuper = mock(ISupervisor.class);
             StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
-                    containerLauncher, "localhost", port, iSuper, state);
+                    containerLauncher, "localhost", port, iSuper, state, cb);
             DynamicState dynamicState = new DynamicState(null, null, null)
                     .withNewAssignment(newAssignment);
-            
+
             DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
-            verify(localizer).requestDownloadBaseTopologyBlobs(newAssignment, port);
-            assertEquals(MachineState.WAITING_FOR_BASIC_LOCALIZATION, nextState.state);
-            assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
-            assertEquals(newAssignment, nextState.pendingLocalization);
-            assertEquals(0, Time.currentTimeMillis());
-            
-            nextState = Slot.stateMachineStep(nextState, staticState);
-            verify(baseFuture).get(1000, TimeUnit.MILLISECONDS);
-            verify(localizer).requestDownloadTopologyBlobs(newAssignment, port);
+            verify(localizer).requestDownloadTopologyBlobs(newAssignment, port, cb);
             assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
             assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
             assertEquals(newAssignment, nextState.pendingLocalization);
@@ -221,6 +238,7 @@ public class SlotTest {
                     mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
             
             AsyncLocalizer localizer = mock(AsyncLocalizer.class);
+            BlobChangingCallback cb = mock(BlobChangingCallback.class);
             Container container = mock(Container.class);
             ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
             LSWorkerHeartbeat oldhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs()-10);
@@ -231,7 +249,7 @@ public class SlotTest {
             ISupervisor iSuper = mock(ISupervisor.class);
             LocalState state = mock(LocalState.class);
             StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
-                    containerLauncher, "localhost", port, iSuper, state);
+                    containerLauncher, "localhost", port, iSuper, state, cb);
             DynamicState dynamicState = new DynamicState(assignment, container, assignment);
             
             DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
@@ -265,7 +283,9 @@ public class SlotTest {
             List<ExecutorInfo> cExecList =  mkExecutorInfoList(1,2,3,4,5);
             LocalAssignment cAssignment = 
                     mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
-            
+
+            BlobChangingCallback cb = mock(BlobChangingCallback.class);
+
             Container cContainer = mock(Container.class);
             LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
             when(cContainer.readHeartbeat()).thenReturn(chb);
@@ -285,48 +305,36 @@ public class SlotTest {
             when(nContainer.readHeartbeat()).thenReturn(nhb, nhb);
             
             @SuppressWarnings("unchecked")
-            CompletableFuture<Void> baseFuture = mock(CompletableFuture.class);
-            when(localizer.requestDownloadBaseTopologyBlobs(nAssignment, port)).thenReturn(baseFuture);
-            
-            @SuppressWarnings("unchecked")
             CompletableFuture<Void> blobFuture = mock(CompletableFuture.class);
-            when(localizer.requestDownloadTopologyBlobs(nAssignment, port)).thenReturn(blobFuture);
+            when(localizer.requestDownloadTopologyBlobs(nAssignment, port, cb)).thenReturn(blobFuture);
             
             ISupervisor iSuper = mock(ISupervisor.class);
             StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
-                    containerLauncher, "localhost", port, iSuper, state);
+                    containerLauncher, "localhost", port, iSuper, state, cb);
             DynamicState dynamicState = new DynamicState(cAssignment, cContainer, nAssignment);
             
             DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
             assertEquals(MachineState.KILL, nextState.state);
             verify(cContainer).kill();
-            verify(localizer).requestDownloadBaseTopologyBlobs(nAssignment, port);
-            assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
+            verify(localizer).requestDownloadTopologyBlobs(nAssignment, port, cb);
+            assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
             assertEquals(nAssignment, nextState.pendingLocalization);
             assertTrue(Time.currentTimeMillis() > 1000);
             
             nextState = Slot.stateMachineStep(nextState, staticState);
             assertEquals(MachineState.KILL, nextState.state);
             verify(cContainer).forceKill();
-            assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
+            assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
             assertEquals(nAssignment, nextState.pendingLocalization);
             assertTrue(Time.currentTimeMillis() > 2000);
             
             nextState = Slot.stateMachineStep(nextState, staticState);
-            assertEquals(MachineState.WAITING_FOR_BASIC_LOCALIZATION, nextState.state);
+            assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
             verify(cContainer).cleanUp();
             verify(localizer).releaseSlotFor(cAssignment, port);
             assertTrue(Time.currentTimeMillis() > 2000);
             
             nextState = Slot.stateMachineStep(nextState, staticState);
-            assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
-            verify(baseFuture).get(1000, TimeUnit.MILLISECONDS);
-            verify(localizer).requestDownloadTopologyBlobs(nAssignment, port);
-            assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
-            assertEquals(nAssignment, nextState.pendingLocalization);
-            assertTrue(Time.currentTimeMillis() > 2000);
-            
-            nextState = Slot.stateMachineStep(nextState, staticState);
             verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
             verify(containerLauncher).launchContainer(port, nAssignment, state);
             assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
@@ -361,7 +369,6 @@ public class SlotTest {
             assertTrue(Time.currentTimeMillis() > 4000);
         }
     }
-
     
     @Test
     public void testRunningToEmpty() throws Exception {
@@ -378,18 +385,19 @@ public class SlotTest {
             when(cContainer.areAllProcessesDead()).thenReturn(false, true);
             
             AsyncLocalizer localizer = mock(AsyncLocalizer.class);
+            BlobChangingCallback cb = mock(BlobChangingCallback.class);
             ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
             
             ISupervisor iSuper = mock(ISupervisor.class);
             LocalState state = mock(LocalState.class);
             StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
-                    containerLauncher, "localhost", port, iSuper, state);
+                    containerLauncher, "localhost", port, iSuper, state, cb);
             DynamicState dynamicState = new DynamicState(cAssignment, cContainer, null);
             
             DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
             assertEquals(MachineState.KILL, nextState.state);
             verify(cContainer).kill();
-            verify(localizer, never()).requestDownloadBaseTopologyBlobs(null, port);
+            verify(localizer, never()).requestDownloadTopologyBlobs(null, port, cb);
             assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
             assertEquals(null, nextState.pendingLocalization);
             assertTrue(Time.currentTimeMillis() > 1000);
@@ -438,12 +446,13 @@ public class SlotTest {
             when(cContainer.runProfiling(any(ProfileRequest.class), anyBoolean())).thenReturn(true);
             
             AsyncLocalizer localizer = mock(AsyncLocalizer.class);
+            BlobChangingCallback cb = mock(BlobChangingCallback.class);
             ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
             
             ISupervisor iSuper = mock(ISupervisor.class);
             LocalState state = mock(LocalState.class);
             StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
-                    containerLauncher, "localhost", port, iSuper, state);
+                    containerLauncher, "localhost", port, iSuper, state, cb);
             Set<TopoProfileAction> profileActions = new HashSet<>();
             ProfileRequest request = new ProfileRequest();
             request.set_action(ProfileAction.JPROFILE_STOP);
@@ -495,4 +504,103 @@ public class SlotTest {
             assertTrue(Time.currentTimeMillis() > 5000);
         }
     }
+
+    @Test
+    public void testResourcesChanged() throws Exception {
+        try (SimulatedTime t = new SimulatedTime(1010)){
+            int port = 8080;
+            String cTopoId = "CURRENT";
+            List<ExecutorInfo> cExecList =  mkExecutorInfoList(1,2,3,4,5);
+            LocalAssignment cAssignment =
+                mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
+
+            BlobChangingCallback cb = mock(BlobChangingCallback.class);
+
+            Container cContainer = mock(Container.class);
+            LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
+            when(cContainer.readHeartbeat()).thenReturn(chb);
+            when(cContainer.areAllProcessesDead()).thenReturn(false, true);
+
+            AsyncLocalizer localizer = mock(AsyncLocalizer.class);
+            Container nContainer = mock(Container.class);
+            LocalState state = mock(LocalState.class);
+            ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+            when(containerLauncher.launchContainer(port, cAssignment, state)).thenReturn(nContainer);
+            when(nContainer.readHeartbeat()).thenReturn(chb, chb);
+
+            ISupervisor iSuper = mock(ISupervisor.class);
+            StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+                containerLauncher, "localhost", port, iSuper, state, cb);
+
+            Set<Slot.BlobChangeing> changing = new HashSet<>();
+            LocallyCachedBlob stormJar = mock(LocallyCachedBlob.class);
+            GoodToGo.GoodToGoLatch stormJarLatch = mock(GoodToGo.GoodToGoLatch.class);
+            CompletableFuture<Void> stormJarLatchFuture = mock(CompletableFuture.class);
+            when(stormJarLatch.countDown()).thenReturn(stormJarLatchFuture);
+            changing.add(new Slot.BlobChangeing(cAssignment, stormJar, stormJarLatch));
+
+            DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment).withChangingBlobs(changing);
+
+            DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+            assertEquals(MachineState.KILL_BLOB_UPDATE, nextState.state);
+            verify(iSuper).killedWorker(port);
+            verify(cContainer).kill();
+            verify(localizer, never()).requestDownloadTopologyBlobs(any(), anyInt(), any());
+            assertNull(nextState.pendingDownload);
+            assertNull(nextState.pendingLocalization);
+            assertEquals(changing, nextState.changingBlobs);
+            assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+            assertNull(nextState.pendingChangingBlobsAssignment);
+            assertTrue(Time.currentTimeMillis() > 1000);
+
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.KILL_BLOB_UPDATE, nextState.state);
+            verify(cContainer).forceKill();
+            assertNull(nextState.pendingDownload);
+            assertNull(nextState.pendingLocalization);
+            assertEquals(changing, nextState.changingBlobs);
+            assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+            assertNull(nextState.pendingChangingBlobsAssignment);
+            assertTrue(Time.currentTimeMillis() > 2000);
+
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.WAITING_FOR_BLOB_UPDATE, nextState.state);
+            verify(cContainer).cleanUp();
+            assertTrue(Time.currentTimeMillis() > 2000);
+
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            verify(stormJarLatchFuture).get(anyLong(), any());
+            verify(containerLauncher).launchContainer(port, cAssignment, state);
+            assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
+            assertNull(nextState.pendingChangingBlobsAssignment);
+            assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+            assertSame(cAssignment, nextState.currentAssignment);
+            assertSame(nContainer, nextState.container);
+            assertTrue(Time.currentTimeMillis() > 2000);
+
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertNull(nextState.pendingChangingBlobsAssignment);
+            assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+            assertSame(cAssignment, nextState.currentAssignment);
+            assertSame(nContainer, nextState.container);
+            assertTrue(Time.currentTimeMillis() > 2000);
+
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertNull(nextState.pendingChangingBlobsAssignment);
+            assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+            assertSame(cAssignment, nextState.currentAssignment);
+            assertSame(nContainer, nextState.container);
+            assertTrue(Time.currentTimeMillis() > 3000);
+
+            nextState = Slot.stateMachineStep(nextState, staticState);
+            assertEquals(MachineState.RUNNING, nextState.state);
+            assertNull(nextState.pendingChangingBlobsAssignment);
+            assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+            assertSame(cAssignment, nextState.currentAssignment);
+            assertSame(nContainer, nextState.container);
+            assertTrue(Time.currentTimeMillis() > 4000);
+        }
+    }
 }