You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2017/10/10 18:48:24 UTC
[2/8] storm git commit: STORM-2438: added in rebalance changes to
support RAS
http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/LocalDownloadedResource.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalDownloadedResource.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalDownloadedResource.java
index f019374..2d2e879 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocalDownloadedResource.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalDownloadedResource.java
@@ -15,59 +15,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.localizer;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
import org.apache.storm.generated.LocalAssignment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * Used for accounting to keep track of who is waiting for specific resources to be downloaded.
+ */
public class LocalDownloadedResource {
private static final Logger LOG = LoggerFactory.getLogger(LocalDownloadedResource.class);
- private static class PortNAssignment {
- private final int _port;
- private final LocalAssignment _assignment;
-
- public PortNAssignment(int port, LocalAssignment assignment) {
- _port = port;
- _assignment = assignment;
- }
-
- @Override
- public boolean equals(Object other) {
- if (!(other instanceof PortNAssignment)) {
- return false;
- }
- PortNAssignment pna = (PortNAssignment) other;
- return pna._port == _port && _assignment.equals(pna._assignment);
- }
-
- @Override
- public int hashCode() {
- return (17 * _port) + _assignment.hashCode();
- }
-
- @Override
- public String toString() {
- return "{"+ _port + " " + _assignment +"}";
- }
- }
- private final CompletableFuture<Void> _pending;
- private final Set<PortNAssignment> _references;
- private boolean _isDone;
+ private final CompletableFuture<Void> pending;
+ private final Set<PortAndAssignment> references;
+ private boolean isDone;
public LocalDownloadedResource(CompletableFuture<Void> pending) {
- _pending = pending;
- _references = new HashSet<>();
- _isDone = false;
+ this.pending = pending;
+ references = new HashSet<>();
+ isDone = false;
}
/**
@@ -77,11 +49,11 @@ public class LocalDownloadedResource {
* @return a future that can be used to track it being downloaded.
*/
public synchronized CompletableFuture<Void> reserve(int port, LocalAssignment la) {
- PortNAssignment pna = new PortNAssignment(port, la);
- if (!_references.add(pna)) {
- LOG.warn("Resources {} already reserved {} for this topology", pna, _references);
+ PortAndAssignment pna = new PortAndAssignment(port, la);
+ if (!references.add(pna)) {
+ LOG.warn("Resources {} already reserved {} for this topology", pna, references);
}
- return _pending;
+ return pending;
}
/**
@@ -91,10 +63,10 @@ public class LocalDownloadedResource {
* @return true if the port was being counted else false
*/
public synchronized boolean release(int port, LocalAssignment la) {
- PortNAssignment pna = new PortNAssignment(port, la);
- boolean ret = _references.remove(pna);
- if (ret && _references.isEmpty()) {
- _isDone = true;
+ PortAndAssignment pna = new PortAndAssignment(port, la);
+ boolean ret = references.remove(pna);
+ if (ret && references.isEmpty()) {
+ isDone = true;
}
return ret;
}
@@ -104,11 +76,11 @@ public class LocalDownloadedResource {
* @return true if it is done else false
*/
public synchronized boolean isDone() {
- return _isDone;
+ return isDone;
}
@Override
public String toString() {
- return _references.toString();
+ return references.toString();
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
index b6b5cd1..7241976 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResource.java
@@ -127,4 +127,17 @@ public class LocalizedResource {
return System.nanoTime();
}
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof LocalizedResource) {
+ LocalizedResource l = (LocalizedResource)other;
+ return _key.equals(l._key) && _uncompressed == l._uncompressed && _localPath.equals(l._localPath);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return _key.hashCode() + Boolean.hashCode(_uncompressed) + _localPath.hashCode();
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
index 9f42b47..826bf98 100644
--- a/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocalizedResourceRetentionSet.java
@@ -17,6 +17,9 @@
*/
package org.apache.storm.localizer;
+import com.google.common.annotations.VisibleForTesting;
+import java.nio.file.Path;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,112 +32,325 @@ import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
+
/**
* A set of resources that we can look at to see which ones we retain and which ones should be
* removed.
*/
public class LocalizedResourceRetentionSet {
- public static final Logger LOG = LoggerFactory.getLogger(LocalizedResourceRetentionSet.class);
- private long _delSize;
- private long _currentSize;
- // targetSize in Bytes
- private long _targetSize;
- private final SortedMap<LocalizedResource, LocalizedResourceSet> _noReferences;
-
- LocalizedResourceRetentionSet(long targetSize) {
- this(targetSize, new LRUComparator());
- }
-
- LocalizedResourceRetentionSet(long targetSize, Comparator<? super LocalizedResource> cmp) {
- this(targetSize, new TreeMap<LocalizedResource, LocalizedResourceSet>(cmp));
- }
-
- LocalizedResourceRetentionSet(long targetSize,
- SortedMap<LocalizedResource, LocalizedResourceSet> retain) {
- this._noReferences = retain;
- this._targetSize = targetSize;
- }
-
- // for testing
- protected int getSizeWithNoReferences() {
- return _noReferences.size();
- }
-
- protected void addResourcesForSet(Iterator<LocalizedResource> setIter, LocalizedResourceSet set) {
- for (Iterator<LocalizedResource> iter = setIter; setIter.hasNext(); ) {
- LocalizedResource lrsrc = iter.next();
- _currentSize += lrsrc.getSize();
- if (lrsrc.getRefCount() > 0) {
- // always retain resources in use
- continue;
- }
- LOG.debug("adding {} to be checked for cleaning", lrsrc.getKey());
- _noReferences.put(lrsrc, set);
- }
- }
-
- public void addResources(LocalizedResourceSet set) {
- addResourcesForSet(set.getLocalFilesIterator(), set);
- addResourcesForSet(set.getLocalArchivesIterator(), set);
- }
-
- public void cleanup() {
- LOG.debug("cleanup target size: {} current size is: {}", _targetSize, _currentSize);
- for (Iterator<Map.Entry<LocalizedResource, LocalizedResourceSet>> i =
- _noReferences.entrySet().iterator();
- _currentSize - _delSize > _targetSize && i.hasNext();) {
- Map.Entry<LocalizedResource, LocalizedResourceSet> rsrc = i.next();
- LocalizedResource resource = rsrc.getKey();
- LocalizedResourceSet set = rsrc.getValue();
- if (resource != null && set.remove(resource)) {
- if (deleteResource(resource)) {
- _delSize += resource.getSize();
- LOG.info("deleting: " + resource.getFilePath() + " size of: " + resource.getSize());
- i.remove();
- } else {
- // since it failed to delete add it back so it gets retried
- set.add(resource.getKey(), resource, resource.isUncompressed());
- }
- }
- }
- }
-
- protected boolean deleteResource(LocalizedResource resource){
- try {
- String fileWithVersion = resource.getFilePathWithVersion();
- String currentSymlinkName = resource.getCurrentSymlinkPath();
- String versionFile = resource.getVersionFilePath();
- File deletePath = new File(fileWithVersion);
- if (resource.isUncompressed()) {
- // this doesn't follow symlinks, which is what we want
- FileUtils.deleteDirectory(deletePath);
- } else {
- Files.delete(deletePath.toPath());
- }
- Files.delete(new File(currentSymlinkName).toPath());
- Files.delete(new File(versionFile).toPath());
- return true;
- } catch (IOException e) {
- LOG.warn("Could not delete: {}", resource.getFilePath());
- }
- return false;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("Cache: ").append(_currentSize).append(", ");
- sb.append("Deleted: ").append(_delSize);
- return sb.toString();
- }
-
- static class LRUComparator implements Comparator<LocalizedResource> {
- public int compare(LocalizedResource r1, LocalizedResource r2) {
- long ret = r1.getLastAccessTime() - r2.getLastAccessTime();
- if (0 == ret) {
- return System.identityHashCode(r1) - System.identityHashCode(r2);
- }
- return ret > 0 ? 1 : -1;
- }
- }
+ public static final Logger LOG = LoggerFactory.getLogger(LocalizedResourceRetentionSet.class);
+ private long delSize;
+ private long currentSize;
+ // targetSize in Bytes
+ private long targetSize;
+ @VisibleForTesting
+ final SortedMap<ComparableResource, CleanableResourceSet> noReferences;
+ private int resourceCount = 0;
+
+ LocalizedResourceRetentionSet(long targetSize) {
+ this(targetSize, new LRUComparator());
+ }
+
+ LocalizedResourceRetentionSet(long targetSize, Comparator<? super ComparableResource> cmp) {
+ this(targetSize, new TreeMap<>(cmp));
+ }
+
+ LocalizedResourceRetentionSet(long targetSize,
+ SortedMap<ComparableResource, CleanableResourceSet> retain) {
+ this.noReferences = retain;
+ this.targetSize = targetSize;
+ }
+
+ // for testing
+ protected int getSizeWithNoReferences() {
+ return noReferences.size();
+ }
+
+ protected void addResourcesForSet(Iterator<LocalizedResource> setIter, LocalizedResourceSet set) {
+ CleanableLocalizedResourceSet cleanset = new CleanableLocalizedResourceSet(set);
+ for (Iterator<LocalizedResource> iter = setIter; setIter.hasNext(); ) {
+ LocalizedResource lrsrc = iter.next();
+ currentSize += lrsrc.getSize();
+ resourceCount ++;
+ if (lrsrc.getRefCount() > 0) {
+ // always retain resources in use
+ continue;
+ }
+ noReferences.put(new LocalizedBlobComparableResource(lrsrc), cleanset);
+ }
+ }
+
+ public void addResources(LocalizedResourceSet set) {
+ addResourcesForSet(set.getLocalFilesIterator(), set);
+ addResourcesForSet(set.getLocalArchivesIterator(), set);
+ }
+
+ public void addResources(ConcurrentHashMap<String, LocallyCachedBlob> blobs) {
+ CleanableLocalizedLocallyCachedBlob set = new CleanableLocalizedLocallyCachedBlob(blobs);
+ for (LocallyCachedBlob b: blobs.values()) {
+ currentSize += b.getSizeOnDisk();
+ resourceCount ++;
+ if (b.isUsed()) {
+ // always retain resources in use
+ continue;
+ }
+ LocallyCachedBlobComparableResource cb = new LocallyCachedBlobComparableResource(b);
+ noReferences.put(cb, set);
+ }
+ }
+
+ public void cleanup() {
+ LOG.debug("cleanup target size: {} current size is: {}", targetSize, currentSize);
+ for (Iterator<Map.Entry<ComparableResource, CleanableResourceSet>> i =
+ noReferences.entrySet().iterator();
+ currentSize - delSize > targetSize && i.hasNext();) {
+ Map.Entry<ComparableResource, CleanableResourceSet> rsrc = i.next();
+ ComparableResource resource = rsrc.getKey();
+ CleanableResourceSet set = rsrc.getValue();
+ if (resource != null && set.remove(resource)) {
+ if (set.deleteUnderlyingResource(resource)) {
+ delSize += resource.getSize();
+ LOG.info("deleting: {} with size of: {}", resource.getNameForDebug(), resource.getSize());
+ i.remove();
+ } else {
+ // since it failed to delete add it back so it gets retried
+ set.add(resource.getKey(), resource);
+ }
+ }
+ }
+ }
+
+ @VisibleForTesting
+ public boolean deleteResource(CleanableResourceSet set, ComparableResource resource) {
+ return set.deleteUnderlyingResource(resource);
+ }
+
+ public long getCurrentSize() {
+ return currentSize;
+ }
+
+ public int getResourceCount() {
+ return resourceCount;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Cache: ").append(currentSize).append(", ");
+ sb.append("Deleted: ").append(delSize);
+ return sb.toString();
+ }
+
+ interface ComparableResource {
+ long getLastAccessTime();
+
+ long getSize();
+
+ String getNameForDebug();
+
+ String getKey();
+ }
+
+ interface CleanableResourceSet {
+ boolean remove(ComparableResource resource);
+
+ void add(String key, ComparableResource resource);
+
+ boolean deleteUnderlyingResource(ComparableResource resource);
+ }
+
+ public static class LocallyCachedBlobComparableResource implements ComparableResource {
+ private final LocallyCachedBlob blob;
+
+ public LocallyCachedBlobComparableResource(LocallyCachedBlob blob) {
+ this.blob = blob;
+ }
+
+ @Override
+ public long getLastAccessTime() {
+ return blob.getLastUsed();
+ }
+
+ @Override
+ public long getSize() {
+ return blob.getSizeOnDisk();
+ }
+
+ @Override
+ public String getNameForDebug() {
+ return blob.getKey();
+ }
+
+ @Override
+ public String getKey() {
+ return blob.getKey();
+ }
+
+ @Override
+ public String toString() {
+ return blob.toString();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof LocallyCachedBlobComparableResource) {
+ return blob.equals(((LocallyCachedBlobComparableResource) other).blob);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return blob.hashCode();
+ }
+ }
+
+ private static class CleanableLocalizedLocallyCachedBlob implements CleanableResourceSet {
+ private final ConcurrentHashMap<String, LocallyCachedBlob> blobs;
+
+ public CleanableLocalizedLocallyCachedBlob(ConcurrentHashMap<String, LocallyCachedBlob> blobs) {
+ this.blobs = blobs;
+ }
+
+ @Override
+ public boolean remove(ComparableResource resource) {
+ if (!(resource instanceof LocallyCachedBlobComparableResource)) {
+ throw new IllegalStateException(resource + " must be a LocallyCachedBlobComparableResource");
+ }
+ LocallyCachedBlob blob = ((LocallyCachedBlobComparableResource)resource).blob;
+ synchronized (blob) {
+ if (!blob.isUsed()) {
+ try {
+ blob.completelyRemove();
+ } catch (Exception e) {
+ LOG.warn("Tried to remove {} but failed with", blob, e);
+ }
+ blobs.remove(blob.getKey());
+ return true;
+ }
+ return false;
+ }
+ }
+
+ @Override
+ public void add(String key, ComparableResource resource) {
+ ///NOOP not used
+ }
+
+ @Override
+ public boolean deleteUnderlyingResource(ComparableResource resource) {
+ //NOOP not used
+ return true;
+ }
+ }
+
+ private static class LocalizedBlobComparableResource implements ComparableResource {
+ private final LocalizedResource resource;
+
+ private LocalizedBlobComparableResource(LocalizedResource resource) {
+ this.resource = resource;
+ }
+
+ @Override
+ public long getLastAccessTime() {
+ return resource.getLastAccessTime();
+ }
+
+ @Override
+ public long getSize() {
+ return resource.getSize();
+ }
+
+ @Override
+ public String getNameForDebug() {
+ return resource.getFilePath();
+ }
+
+ @Override
+ public String getKey() {
+ return resource.getKey();
+ }
+
+ @Override
+ public String toString() {
+ return resource.getKey() + " at " + resource.getFilePathWithVersion();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof LocalizedBlobComparableResource) {
+ return resource.equals(((LocalizedBlobComparableResource) other).resource);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return resource.hashCode();
+ }
+ }
+
+ private static class CleanableLocalizedResourceSet implements CleanableResourceSet {
+ private final LocalizedResourceSet set;
+
+ public CleanableLocalizedResourceSet(LocalizedResourceSet set) {
+ this.set = set;
+ }
+
+ @Override
+ public boolean remove(ComparableResource resource) {
+ if (!(resource instanceof LocalizedBlobComparableResource)) {
+ throw new IllegalStateException(resource + " must be a LocalizedBlobComparableResource");
+ }
+ return set.remove(((LocalizedBlobComparableResource)resource).resource);
+ }
+
+ @Override
+ public void add(String key, ComparableResource resource) {
+ if (!(resource instanceof LocalizedBlobComparableResource)) {
+ throw new IllegalStateException(resource + " must be a LocalizedBlobComparableResource");
+ }
+ LocalizedResource r = ((LocalizedBlobComparableResource)resource).resource;
+ set.add(key, r, r.isUncompressed());
+ }
+
+ @Override
+ public boolean deleteUnderlyingResource(ComparableResource resource) {
+ if (resource instanceof LocalizedBlobComparableResource) {
+ LocalizedResource lr = ((LocalizedBlobComparableResource) resource).resource;
+ try {
+ Path fileWithVersion = new File(lr.getFilePathWithVersion()).toPath();
+ Path currentSymLink = new File(lr.getCurrentSymlinkPath()).toPath();
+ Path versionFile = new File(lr.getVersionFilePath()).toPath();
+
+ if (lr.isUncompressed()) {
+ if (Files.exists(fileWithVersion)) {
+ // this doesn't follow symlinks, which is what we want
+ FileUtils.deleteDirectory(fileWithVersion.toFile());
+ }
+ } else {
+ Files.deleteIfExists(fileWithVersion);
+ }
+ Files.deleteIfExists(currentSymLink);
+ Files.deleteIfExists(versionFile);
+ return true;
+ } catch (IOException e) {
+ LOG.warn("Could not delete: {}", resource.getNameForDebug(), e);
+ }
+ return false;
+ } else {
+ throw new IllegalArgumentException("Don't know how to handle a " + resource.getClass());
+ }
+ }
+ }
+
+ static class LRUComparator implements Comparator<ComparableResource> {
+ public int compare(ComparableResource r1, ComparableResource r2) {
+ long ret = r1.getLastAccessTime() - r2.getLastAccessTime();
+ if (0 == ret) {
+ return System.identityHashCode(r1) - System.identityHashCode(r2);
+ }
+ return ret > 0 ? 1 : -1;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java
new file mode 100644
index 0000000..c09108d
--- /dev/null
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedBlob.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.localizer;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.storm.blobstore.BlobStore;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.KeyNotFoundException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents a blob that is cached locally on disk by the supervisor.
+ */
+public abstract class LocallyCachedBlob {
+ private static final Logger LOG = LoggerFactory.getLogger(LocallyCachedBlob.class);
+ public static final long NOT_DOWNLOADED_VERSION = -1;
+ // A callback that does nothing.
+ private static final BlobChangingCallback NOOP_CB = (assignment, port, resource, go) -> {};
+
+ private long lastUsed = System.currentTimeMillis();
+ private final Map<PortAndAssignment, BlobChangingCallback> references = new HashMap<>();
+ private final String blobDescription;
+ private final String blobKey;
+ private CompletableFuture<Void> doneUpdating = null;
+
+ /**
+ * Create a new LocallyCachedBlob.
+ * @param blobDescription a description of the blob this represents. Typically it should at least be the blob key, but ideally also
+ * include if it is an archive or not, what user or topology it is for, or if it is a storm.jar etc.
+ */
+ protected LocallyCachedBlob(String blobDescription, String blobKey) {
+ this.blobDescription = blobDescription;
+ this.blobKey = blobKey;
+ }
+
+ /**
+ * Get the version of the blob cached locally. If the version is unknown or it has not been downloaded NOT_DOWNLOADED_VERSION
+ * should be returned.
+ * PRECONDITION: this can only be called with a lock on this instance held.
+ */
+ public abstract long getLocalVersion();
+
+ /**
+ * Get the version of the blob in the blob store.
+ * PRECONDITION: this can only be called with a lock on this instance held.
+ */
+ public abstract long getRemoteVersion(ClientBlobStore store) throws KeyNotFoundException, AuthorizationException;
+
+ /**
+ * Download the latest version to a temp location. This may also include unzipping some or all of the data to a temp location.
+ * PRECONDITION: this can only be called with a lock on this instance held.
+ * @param store the store to us to download the data.
+ * @return the version that was downloaded.
+ */
+ public abstract long downloadToTempLocation(ClientBlobStore store) throws IOException, KeyNotFoundException, AuthorizationException;
+
+ /**
+ * Commit the new version and make it available for the end user.
+ * PRECONDITION: uncompressToTempLocationIfNeeded will have been called.
+ * PRECONDITION: this can only be called with a lock on this instance held.
+ * @param version the version of the blob to commit.
+ */
+ public abstract void commitNewVersion(long version) throws IOException;
+
+ /**
+ * Clean up any temporary files. This will be called after updating a blob, either successfully or if an error has occured.
+ * The goal is to find any files that may be left over and remove them so space is not leaked.
+ * PRECONDITION: this can only be called with a lock on this instance held.
+ */
+ public abstract void cleanupOrphanedData() throws IOException;
+
+ /**
+ * Completely remove anything that is cached locally for this blob and all tracking files also stored for it.
+ * This will be called after the blob was determined to no longer be needed in the cache.
+ * PRECONDITION: this can only be called with a lock on this instance held.
+ */
+ public abstract void completelyRemove() throws IOException;
+
+ /**
+ * Get the amount of disk space that is used by this blob. If the blob is uncompressed it should be the sum of the space used by all
+ * of the uncompressed files. In general this will not be called with any locks held so it is a good idea to cache it and updated it
+ * when committing a new version.
+ */
+ public abstract long getSizeOnDisk();
+
+ /**
+ * Updates the last updated time. This should be called when references are added or removed.
+ */
+ private synchronized void touch() {
+ lastUsed = System.currentTimeMillis();
+ }
+
+ /**
+ * Get the last time that this used for LRU calculations.
+ */
+ public synchronized long getLastUsed() {
+ return lastUsed;
+ }
+
+ /**
+ * Return true if this blob is actively being used, else false (meaning it can be deleted, but might not be).
+ */
+ public synchronized boolean isUsed() {
+ return !references.isEmpty();
+ }
+
+ /**
+ * Get the size of p in bytes.
+ * @param p the path to read.
+ * @return the size of p in bytes.
+ */
+ protected long getSizeOnDisk(Path p) throws IOException {
+ if (!Files.exists(p)) {
+ return 0;
+ } else if (Files.isRegularFile(p)) {
+ return Files.size(p);
+ } else {
+ //We will not follow sym links
+ return Files.walk(p)
+ .filter((subp) -> Files.isRegularFile(subp, LinkOption.NOFOLLOW_LINKS))
+ .mapToLong((subp) -> {
+ try {
+ return Files.size(subp);
+ } catch (IOException e) {
+ LOG.warn("Could not get the size of ");
+ }
+ return 0;
+ }).sum();
+ }
+ }
+
+ /**
+ * Mark that a given port and assignemnt are using this.
+ * @param pna the slot and assignment that are using this blob.
+ * @param cb an optional callback indicating that they want to know/synchronize when a blob is updated.
+ */
+ public void addReference(final PortAndAssignment pna, BlobChangingCallback cb) {
+ if (cb == null) {
+ cb = NOOP_CB;
+ }
+ if (references.put(pna, cb) != null) {
+ LOG.warn("{} already has a reservation for {}", pna, blobDescription);
+ }
+ }
+
+ /**
+ * Removes a reservation for this blob from a given slot and assignemnt.
+ * @param pna the slot + assignment that no longer needs this blob.
+ */
+ public void removeReference(final PortAndAssignment pna) {
+ if (references.remove(pna) == null) {
+ LOG.warn("{} had no reservation for {}", pna, blobDescription);
+ }
+ }
+
+ /**
+ * Inform all of the callbacks that a change is going to happen and then wait for
+ * them to all get back that it is OK to make that change.
+ */
+ public synchronized void informAllOfChangeAndWaitForConsensus() {
+ CountDownLatch cdl = new CountDownLatch(references.size());
+ doneUpdating = new CompletableFuture<>();
+ for (Map.Entry<PortAndAssignment, BlobChangingCallback> entry : references.entrySet()) {
+ GoodToGo gtg = new GoodToGo(cdl, doneUpdating);
+ try {
+ PortAndAssignment pna = entry.getKey();
+ BlobChangingCallback cb = entry.getValue();
+ //TODO we probably want to not use this, or make it just return something that has less power to modify things
+ cb.blobChanging(pna.getAssignment(), pna.getPort(), this, gtg);
+ } finally {
+ gtg.countDownIfLatchWasNotGotten();
+ }
+ }
+ try {
+ cdl.await(3, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ //TODO need to think about error handling here in general.
+ }
+ }
+
+ /**
+ * Inform all of the callbacks that the change to the blob is complete.
+ */
+ public synchronized void informAllChangeComplete() {
+ doneUpdating.complete(null);
+ }
+
+ /**
+ * Get the key for this blob.
+ */
+ public String getKey() {
+ return blobKey;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java
new file mode 100644
index 0000000..35371b5
--- /dev/null
+++ b/storm-server/src/main/java/org/apache/storm/localizer/LocallyCachedTopologyBlob.java
@@ -0,0 +1,364 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.localizer;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.JarURLConnection;
+import java.net.URL;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Enumeration;
+import java.util.Map;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.storm.blobstore.ClientBlobStore;
+import org.apache.storm.blobstore.InputStreamWithMeta;
+import org.apache.storm.daemon.supervisor.AdvancedFSOps;
+import org.apache.storm.generated.AuthorizationException;
+import org.apache.storm.generated.KeyNotFoundException;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ServerConfigUtils;
+import org.apache.storm.utils.ServerUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A locally cached blob for the topology. storm.jar, stormcode.ser, or stormconf.ser.
+ * The version number of the blob's file will be stored in `${basename}.version`
+ */
+public class LocallyCachedTopologyBlob extends LocallyCachedBlob {
+ private static final Logger LOG = LoggerFactory.getLogger(LocallyCachedTopologyBlob.class);
+ public static final long LOCAL_MODE_JAR_VERSION = 1;
+
+ private static String resourcesJar() throws IOException {
+ String path = ServerUtils.currentClasspath();
+ if (path == null) {
+ return null;
+ }
+
+ for (String jpath : path.split(File.pathSeparator)) {
+ if (jpath.endsWith(".jar")) {
+ if (ServerUtils.zipDoesContainDir(jpath, ServerConfigUtils.RESOURCES_SUBDIR)) {
+ return jpath;
+ }
+ }
+ }
+ return null;
+ }
+
+ public enum TopologyBlobType {
+ TOPO_JAR("stormjar.jar", "-stormjar.jar", "resources"),
+ TOPO_CODE("stormcode.ser", "-stormcode.ser", null),
+ TOPO_CONF("stormconf.ser", "-stormconf.ser", null);
+
+ private final String fileName;
+ private final String keySuffix;
+ private final String extractionDir;
+
+ TopologyBlobType(String fileName, String keySuffix, String extractionDir) {
+ this.fileName = fileName;
+ this.keySuffix = keySuffix;
+ this.extractionDir = extractionDir;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public String getTempFileName(long version) {
+ return fileName + "." + version;
+ }
+
+ public String getVersionFileName() {
+ return fileName + ".version";
+ }
+
+ public String getKey(String topologyId) {
+ return topologyId + keySuffix;
+ }
+
+ public boolean needsExtraction() {
+ return extractionDir != null;
+ }
+
+ public String getExtractionDir() {
+ return extractionDir;
+ }
+
+ public String getTempExtractionDir(long version) {
+ return extractionDir + "." + version;
+ }
+ };
+
+ private final TopologyBlobType type;
+ private final String topologyId;
+ private final boolean isLocalMode;
+ private final Path topologyBasicBlobsRootDir;
+ private final AdvancedFSOps fsOps;
+ private final Map<String, Object> conf;
+ private volatile long version = NOT_DOWNLOADED_VERSION;
+ private volatile long size = 0;
+
+ /**
+ * Create a new LocallyCachedBlob.
+ *
+ * @param topologyId the ID of the topology.
+ * @param type the type of the blob.
+ */
+ protected LocallyCachedTopologyBlob(final String topologyId, final boolean isLocalMode, final Map<String, Object> conf,
+ final AdvancedFSOps fsOps, final TopologyBlobType type) throws IOException {
+ super(topologyId + " " + type.getFileName(), type.getKey(topologyId));
+ this.topologyId = topologyId;
+ this.type = type;
+ this.isLocalMode = isLocalMode;
+ this.fsOps = fsOps;
+ this.conf = conf;
+ topologyBasicBlobsRootDir = Paths.get(ConfigUtils.supervisorStormDistRoot(conf, topologyId));
+ readVersion();
+ updateSizeOnDisk();
+ }
+
+ private void updateSizeOnDisk() throws IOException {
+ long total = getSizeOnDisk(topologyBasicBlobsRootDir.resolve(type.getFileName()));
+ if (type.needsExtraction()) {
+ total += getSizeOnDisk(topologyBasicBlobsRootDir.resolve(type.getExtractionDir()));
+ }
+ size = total;
+ }
+
+ private void readVersion() throws IOException {
+ Path versionFile = topologyBasicBlobsRootDir.resolve(type.getVersionFileName());
+ if (!fsOps.fileExists(versionFile)) {
+ version = NOT_DOWNLOADED_VERSION;
+ } else {
+ String ver = FileUtils.readFileToString(versionFile.toFile(), "UTF8").trim();
+ version = Long.parseLong(ver);
+ }
+ }
+
+ @Override
+ public long getLocalVersion() {
+ LOG.debug("LOCAL VERSION {}/{} is {}", type, topologyId, version);
+ return version;
+ }
+
+ @Override
+ public long getRemoteVersion(ClientBlobStore store) throws KeyNotFoundException, AuthorizationException {
+ if (isLocalMode && type == TopologyBlobType.TOPO_JAR) {
+ LOG.debug("REMOTE VERSION LOCAL JAR {}", LOCAL_MODE_JAR_VERSION);
+ return LOCAL_MODE_JAR_VERSION;
+ }
+ return store.getBlobMeta(type.getKey(topologyId)).get_version();
+ }
+
+ @Override
+ public long downloadToTempLocation(ClientBlobStore store)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ if (isLocalMode && type == TopologyBlobType.TOPO_JAR) {
+ LOG.debug("DOWNLOADING LOCAL JAR to TEMP LOCATION... {}", topologyId);
+ //This is a special case where the jar was not uploaded so we will not download it (it is already on the classpath)
+ ClassLoader classloader = Thread.currentThread().getContextClassLoader();
+ String resourcesJar = resourcesJar();
+ URL url = classloader.getResource(ServerConfigUtils.RESOURCES_SUBDIR);
+ Path extractionDest = topologyBasicBlobsRootDir.resolve(type.getTempExtractionDir(LOCAL_MODE_JAR_VERSION));
+ if (resourcesJar != null) {
+ LOG.info("Extracting resources from jar at {} to {}", resourcesJar, extractionDest);
+ extractDirFromJar(resourcesJar, ServerConfigUtils.RESOURCES_SUBDIR, extractionDest);
+ } else if (url != null) {
+ LOG.info("Copying resources at {} to {}", url, extractionDest);
+ if ("jar".equals(url.getProtocol())) {
+ JarURLConnection urlConnection = (JarURLConnection) url.openConnection();
+ extractDirFromJar(urlConnection.getJarFileURL().getFile(), ServerConfigUtils.RESOURCES_SUBDIR, extractionDest);
+ } else {
+ fsOps.copyDirectory(new File(url.getFile()), extractionDest.toFile());
+ }
+ }
+ return LOCAL_MODE_JAR_VERSION;
+ }
+
+ long newVersion;
+ Path tmpLocation;
+ String key = type.getKey(topologyId);
+ try (InputStreamWithMeta in = store.getBlob(key)) {
+ newVersion = in.getVersion();
+ long expectedSize = in.getFileLength();
+ if (newVersion == version) {
+ throw new RuntimeException("The version did not change, but we tried to download it. " + version + " " + key);
+ }
+ tmpLocation = topologyBasicBlobsRootDir.resolve(type.getTempFileName(newVersion));
+ long totalRead = 0;
+ //Make sure the parent directory is there and ready to go
+ fsOps.forceMkdir(tmpLocation.getParent());
+ try (OutputStream outStream = fsOps.getOutputStream(tmpLocation.toFile())) {
+ byte [] buffer = new byte[4096];
+ int read = 0;
+ while ((read = in.read(buffer)) > 0) {
+ outStream.write(buffer, 0, read);
+ totalRead += read;
+ }
+ }
+ if (totalRead != expectedSize) {
+ throw new IOException("We expected to download " + expectedSize + " bytes but found we got " + totalRead);
+ }
+ }
+
+ if (type.needsExtraction()) {
+ Path extractionDest = topologyBasicBlobsRootDir.resolve(type.getTempExtractionDir(newVersion));
+ extractDirFromJar(tmpLocation.toAbsolutePath().toString(), ServerConfigUtils.RESOURCES_SUBDIR,
+ extractionDest);
+ }
+ return newVersion;
+ }
+
+ protected void extractDirFromJar(String jarpath, String dir, Path dest) throws IOException {
+ LOG.debug("EXTRACTING {} from {} and placing it at {}", dir, jarpath, dest);
+ try (JarFile jarFile = new JarFile(jarpath)) {
+ String toRemove = dir + '/';
+ Enumeration<JarEntry> jarEnums = jarFile.entries();
+ while (jarEnums.hasMoreElements()) {
+ JarEntry entry = jarEnums.nextElement();
+ String name = entry.getName();
+ if (!entry.isDirectory() && name.startsWith(toRemove)) {
+ String shortenedName = name.replace(toRemove, "");
+ Path aFile = dest.resolve(shortenedName);
+ LOG.debug("EXTRACTING {} SHORTENED to {} into {}", name, shortenedName, aFile);
+ fsOps.forceMkdir(aFile.getParent());
+ try (FileOutputStream out = new FileOutputStream(aFile.toFile());
+ InputStream in = jarFile.getInputStream(entry)) {
+ IOUtils.copy(in, out);
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void commitNewVersion(long newVersion) throws IOException {
+ //This is not atomic (so if something bad happens in the middle we need to be able to recover
+ Path tempLoc = topologyBasicBlobsRootDir.resolve(type.getTempFileName(newVersion));
+ Path dest = topologyBasicBlobsRootDir.resolve(type.getFileName());
+ Path versionFile = topologyBasicBlobsRootDir.resolve(type.getVersionFileName());
+
+ LOG.debug("Removing version file {} to force download on failure", versionFile);
+ fsOps.deleteIfExists(versionFile.toFile()); //So if we fail we are forced to try again
+ LOG.debug("Removing destination file {} in preparation for move", dest);
+ fsOps.deleteIfExists(dest.toFile());
+ if (type.needsExtraction()) {
+ Path extractionTemp = topologyBasicBlobsRootDir.resolve(type.getTempExtractionDir(newVersion));
+ Path extractionDest = topologyBasicBlobsRootDir.resolve(type.getExtractionDir());
+ LOG.debug("Removing extraction dest {} in preparation for extraction", extractionDest);
+ fsOps.deleteIfExists(extractionDest.toFile());
+ if (fsOps.fileExists(extractionTemp)) {
+ fsOps.moveDirectoryPreferAtomic(extractionTemp.toFile(), extractionDest.toFile());
+ }
+ }
+ if (!(isLocalMode && type == TopologyBlobType.TOPO_JAR)) {
+ //Don't try to move the JAR file in local mode, it does not exist because it was not uploaded
+ Files.move(tempLoc, dest);
+ }
+ LOG.debug("Writing out version file {} with version {}", versionFile, newVersion);
+ FileUtils.write(versionFile.toFile(), Long.toString(newVersion), "UTF8");
+ this.version = newVersion;
+ updateSizeOnDisk();
+ LOG.debug("New version of {} - {} committed {}", topologyId, type, newVersion);
+ }
+
+ @Override
+ public void cleanupOrphanedData() throws IOException {
+ cleanUpTemp(type.getFileName());
+ if (type.needsExtraction()) {
+ cleanUpTemp(type.getExtractionDir());
+ }
+ }
+
+ private static final Pattern EXTRACT_BASE_NAME_AND_VERSION = Pattern.compile("^(.*)\\.([0-9]+)$");
+
+ private void cleanUpTemp(String baseName) throws IOException {
+ LOG.debug("Cleaning up temporary data in {}", topologyBasicBlobsRootDir);
+ try (DirectoryStream<Path> children = fsOps.newDirectoryStream(topologyBasicBlobsRootDir,
+ (p) -> {
+ String fileName = p.getFileName().toString();
+ Matcher m = EXTRACT_BASE_NAME_AND_VERSION.matcher(fileName);
+ return m.matches() && baseName.equals(m.group(1));
+ })) {
+ //children is only ever null if topologyBasicBlobsRootDir does not exist. This happens during unit tests
+ // And because a non-existant directory is by definition clean we are ignoring it.
+ if (children != null) {
+ for (Path p : children) {
+ LOG.debug("Cleaning up {}", p);
+ fsOps.deleteIfExists(p.toFile());
+ }
+ }
+ }
+ }
+
+ @Override
+ public void completelyRemove() throws IOException {
+ removeAll(type.getFileName());
+ if (type.needsExtraction()) {
+ removeAll(type.getExtractionDir());
+ }
+ }
+
+ private void removeAll(String baseName) throws IOException {
+ try (DirectoryStream<Path> children = fsOps.newDirectoryStream(topologyBasicBlobsRootDir)) {
+ for (Path p : children) {
+ String fileName = p.getFileName().toString();
+ if (fileName.startsWith(baseName)) {
+ fsOps.deleteIfExists(p.toFile());
+ }
+ }
+ }
+ }
+
+ @Override
+ public long getSizeOnDisk() {
+ return size;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof LocallyCachedTopologyBlob) {
+ LocallyCachedTopologyBlob o = (LocallyCachedTopologyBlob)other;
+ return topologyId.equals(o.topologyId) && type == o.type && topologyBasicBlobsRootDir.equals(o.topologyBasicBlobsRootDir);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return topologyId.hashCode() + type.hashCode() + topologyBasicBlobsRootDir.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "LOCAL TOPO BLOB " + type + " " + topologyId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java b/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java
new file mode 100644
index 0000000..081c811
--- /dev/null
+++ b/storm-server/src/main/java/org/apache/storm/localizer/PortAndAssignment.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.localizer;
+
+import org.apache.storm.generated.LocalAssignment;
+
+/**
+ * A Port and a LocalAssignment used to reference count Resources
+ */
+class PortAndAssignment {
+ private final int port;
+ private final LocalAssignment assignment;
+
+ public PortAndAssignment(int port, LocalAssignment assignment) {
+ this.port = port;
+ this.assignment = assignment;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof PortAndAssignment)) {
+ return false;
+ }
+ PortAndAssignment pna = (PortAndAssignment) other;
+ return pna.port == port && assignment.equals(pna.assignment);
+ }
+
+ public String getToplogyId() {
+ return assignment.get_topology_id();
+ }
+
+ @Override
+ public int hashCode() {
+ return (17 * port) + assignment.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "{" + port + " " + assignment + "}";
+ }
+
+ /**
+ * Return the port associated with this.
+ */
+ public int getPort() {
+ return port;
+ }
+
+ /**
+ * return the assigment for this.
+ */
+ public LocalAssignment getAssignment() {
+ return assignment;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
index a50dced..ab8eb14 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/resource/ResourceUtils.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.storm.Config;
import org.apache.storm.generated.Bolt;
+import org.apache.storm.generated.ComponentCommon;
import org.apache.storm.generated.SpoutSpec;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.utils.ObjectReader;
@@ -63,6 +64,71 @@ public class ResourceUtils {
return spoutResources;
}
+ public static void updateStormTopologyResources(StormTopology topology, Map<String, Map<String, Double>> resourceUpdatesMap) {
+ Map<String, Map<String, Double>> componentsUpdated = new HashMap<>();
+ if (topology.get_spouts() != null) {
+ for (Map.Entry<String, SpoutSpec> spout : topology.get_spouts().entrySet()) {
+ SpoutSpec spoutSpec = spout.getValue();
+ String spoutName = spout.getKey();
+
+ if (resourceUpdatesMap.containsKey(spoutName)) {
+ ComponentCommon spoutCommon = spoutSpec.get_common();
+ Map<String, Double> resourcesUpdate = resourceUpdatesMap.get(spoutName);
+ String newJsonConf = getJsonWithUpdatedResources(spoutCommon.get_json_conf(), resourcesUpdate);
+ spoutCommon.set_json_conf(newJsonConf);
+ componentsUpdated.put(spoutName, resourcesUpdate);
+ }
+ }
+ }
+
+ if (topology.get_bolts() != null) {
+ for (Map.Entry<String, Bolt> bolt : topology.get_bolts().entrySet()) {
+ Bolt boltObj = bolt.getValue();
+ String boltName = bolt.getKey();
+
+ if(resourceUpdatesMap.containsKey(boltName)) {
+ ComponentCommon boltCommon = boltObj.get_common();
+ Map<String, Double> resourcesUpdate = resourceUpdatesMap.get(boltName);
+ String newJsonConf = getJsonWithUpdatedResources(boltCommon.get_json_conf(), resourceUpdatesMap.get(boltName));
+ boltCommon.set_json_conf(newJsonConf);
+ componentsUpdated.put(boltName, resourcesUpdate);
+ }
+ }
+ }
+ LOG.info("Component resources updated: {}", componentsUpdated);
+ Map<String, Map<String, Double>> notUpdated = new HashMap<String, Map<String, Double>>();
+ for (String component : resourceUpdatesMap.keySet()) {
+ if (!componentsUpdated.containsKey(component)) {
+ notUpdated.put(component, resourceUpdatesMap.get(component));
+ }
+ }
+ LOG.info("Component resource updates ignored: {}", notUpdated);
+ }
+
+ public static String getJsonWithUpdatedResources(String jsonConf, Map<String, Double> resourceUpdates) {
+ try {
+ JSONParser parser = new JSONParser();
+ Object obj = parser.parse(jsonConf);
+ JSONObject jsonObject = (JSONObject) obj;
+
+ if (resourceUpdates.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB)) {
+ Double topoMemOnHeap = resourceUpdates.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB);
+ jsonObject.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, topoMemOnHeap);
+ }
+ if (resourceUpdates.containsKey(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB)) {
+ Double topoMemOffHeap = resourceUpdates.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB);
+ jsonObject.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, topoMemOffHeap);
+ }
+ if (resourceUpdates.containsKey(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT)) {
+ Double topoCPU = resourceUpdates.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT);
+ jsonObject.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, topoCPU);
+ }
+ return jsonObject.toJSONString();
+ } catch (ParseException ex) {
+ throw new RuntimeException("Failed to parse component resources with json: " + jsonConf);
+ }
+ }
+
public static void checkIntialization(Map<String, Double> topologyResources, String com,
Map<String, Object> topologyConf) {
checkInitMem(topologyResources, com, topologyConf);
http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java b/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
index 9f2b95b..6a4454a 100644
--- a/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/utils/ServerUtils.java
@@ -18,6 +18,7 @@
package org.apache.storm.utils;
+import javax.security.auth.Subject;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.exec.CommandLine;
@@ -32,6 +33,7 @@ import org.apache.storm.blobstore.BlobStoreAclHandler;
import org.apache.storm.blobstore.ClientBlobStore;
import org.apache.storm.blobstore.InputStreamWithMeta;
import org.apache.storm.blobstore.LocalFsBlobStore;
+import org.apache.storm.blobstore.LocalModeClientBlobStore;
import org.apache.storm.daemon.StormCommon;
import org.apache.storm.generated.AccessControl;
import org.apache.storm.generated.AccessControlType;
@@ -43,6 +45,7 @@ import org.apache.storm.generated.SettableBlobMeta;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.nimbus.NimbusInfo;
import org.apache.storm.scheduler.resource.ResourceUtils;
+import org.apache.storm.security.auth.SingleUserPrincipal;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -212,8 +215,13 @@ public class ServerUtils {
}
public static ClientBlobStore getClientBlobStoreForSupervisor(Map<String, Object> conf) {
- ClientBlobStore store = (ClientBlobStore) ReflectionUtils.newInstance(
+ ClientBlobStore store;
+ if (ConfigUtils.isLocalMode(conf)) {
+ store = new LocalModeClientBlobStore(getNimbusBlobStore(conf, null));
+ } else {
+ store = (ClientBlobStore) ReflectionUtils.newInstance(
(String) conf.get(DaemonConfig.SUPERVISOR_BLOBSTORE));
+ }
store.prepare(conf);
return store;
}
@@ -767,4 +775,11 @@ public class ServerUtils {
}
return ret;
}
+
+ public static Subject principalNameToSubject(String name) {
+ SingleUserPrincipal principal = new SingleUserPrincipal(name);
+ Subject sub = new Subject();
+ sub.getPrincipals().add(principal);
+ return sub;
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/test/java/org/apache/storm/TestRebalance.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/TestRebalance.java b/storm-server/src/test/java/org/apache/storm/TestRebalance.java
new file mode 100644
index 0000000..bd72b3a
--- /dev/null
+++ b/storm-server/src/test/java/org/apache/storm/TestRebalance.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm;
+
+import org.apache.storm.generated.ClusterSummary;
+import org.apache.storm.generated.RebalanceOptions;
+import org.apache.storm.generated.StormTopology;
+import org.apache.storm.generated.TopologySummary;
+import org.apache.storm.scheduler.resource.ResourceAwareScheduler;
+import org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler;
+import org.apache.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy;
+import org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy;
+import org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy;
+import org.apache.storm.topology.BoltDeclarer;
+import org.apache.storm.topology.SpoutDeclarer;
+import org.apache.storm.topology.TopologyBuilder;
+import org.apache.storm.utils.Utils;
+import org.apache.thrift.TException;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestRebalance {
+
+ static final int SLEEP_TIME_BETWEEN_RETRY = 1000;
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestRebalance.class);
+
+ @Test
+ public void testRebalanceTopologyResourcesAndConfigs()
+ throws Exception {
+
+ LOG.info("Starting local cluster...");
+
+ Config conf = new Config();
+ conf.put(DaemonConfig.STORM_SCHEDULER, ResourceAwareScheduler.class.getName());
+ conf.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY, DefaultEvictionStrategy.class.getName());
+ conf.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, DefaultSchedulingPriorityStrategy.class.getName());
+ conf.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, DefaultResourceAwareStrategy.class.getName());
+ conf.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 10.0);
+ conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 10.0);
+ conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 100.0);
+ conf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
+
+ try (ILocalCluster cluster = new LocalCluster.Builder().withDaemonConf(conf).build()) {
+
+ TopologyBuilder builder = new TopologyBuilder();
+ SpoutDeclarer s1 = builder.setSpout("spout-1", new TestUtilsForResourceAwareScheduler.TestSpout(),
+ 2);
+ BoltDeclarer b1 = builder.setBolt("bolt-1", new TestUtilsForResourceAwareScheduler.TestBolt(),
+ 2).shuffleGrouping("spout-1");
+ BoltDeclarer b2 = builder.setBolt("bolt-2", new TestUtilsForResourceAwareScheduler.TestBolt(),
+ 2).shuffleGrouping("bolt-1");
+
+ StormTopology stormTopology = builder.createTopology();
+
+ LOG.info("submitting topologies...");
+ String topoName = "topo1";
+ cluster.submitTopology(topoName, new HashMap<>(), stormTopology);
+
+ waitTopologyScheduled(topoName, cluster, 20);
+
+ RebalanceOptions opts = new RebalanceOptions();
+
+ Map<String, Map<String, Double>> resources = new HashMap<String, Map<String, Double>>();
+ resources.put("spout-1", new HashMap<String, Double>());
+ resources.get("spout-1").put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 120.0);
+ resources.get("spout-1").put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 25.0);
+
+ opts.set_topology_resources_overrides(resources);
+ opts.set_wait_secs(0);
+
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 768.0);
+
+ opts.set_topology_conf_overrides(jsonObject.toJSONString());
+
+ LOG.info("rebalancing...");
+ cluster.rebalance("topo1", opts);
+
+ waitTopologyScheduled(topoName, cluster, 10);
+
+ String confRaw = cluster.getTopologyConf(topoNameToId(topoName, cluster));
+
+ JSONParser parser = new JSONParser();
+
+ JSONObject readConf = (JSONObject) parser.parse(confRaw);
+ assertEquals("updated conf correct", 768.0, (double) readConf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB), 0.001);
+
+ StormTopology readStormTopology = cluster.getTopology(topoNameToId(topoName, cluster));
+ String componentConfRaw = readStormTopology.get_spouts().get("spout-1").get_common().get_json_conf();
+
+ JSONObject readTopologyConf = (JSONObject) parser.parse(componentConfRaw);
+
+ assertEquals("Updated CPU correct", 25.0, (double) readTopologyConf.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT), 0.001);
+ assertEquals("Updated Memory correct", 120.0, (double) readTopologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB), 0.001);
+ }
+ }
+
+ public void waitTopologyScheduled(String topoName, ILocalCluster cluster, int retryAttempts) throws TException {
+ for (int i = 0; i < retryAttempts; i++) {
+ if (checkTopologyScheduled(topoName, cluster)) {
+ //sleep to prevent race conditions
+ Utils.sleep(SLEEP_TIME_BETWEEN_RETRY);
+ return;
+ }
+ Utils.sleep(SLEEP_TIME_BETWEEN_RETRY);
+ }
+ throw new RuntimeException("Error: Wait for topology " + topoName + " to be ACTIVE has timed out!");
+ }
+
+ public boolean checkTopologyScheduled(String topoName, ILocalCluster cluster) throws TException {
+ if (checkTopologyUp(topoName, cluster)) {
+ ClusterSummary sum = cluster.getClusterInfo();
+ for (TopologySummary topoSum : sum.get_topologies()) {
+ if (topoSum.get_name().equals(topoName)) {
+ String status = topoSum.get_status();
+ String sched_status = topoSum.get_sched_status();
+ if (status.equals("ACTIVE") && (sched_status != null && !sched_status.equals(""))) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ public boolean checkTopologyUp(String topoName, ILocalCluster cluster) throws TException {
+ ClusterSummary sum = cluster.getClusterInfo();
+
+ for (TopologySummary topoSum : sum.get_topologies()) {
+ if (topoSum.get_name().equals(topoName)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static String topoNameToId(String topoName, ILocalCluster cluster) throws TException {
+ for (TopologySummary topoSum : cluster.getClusterInfo().get_topologies()) {
+ if (topoSum.get_name().equals(topoName)) {
+ return topoSum.get_id();
+ }
+ }
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/e94b735a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
----------------------------------------------------------------------
diff --git a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
index 60b628e..927225d 100644
--- a/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
+++ b/storm-server/src/test/java/org/apache/storm/daemon/supervisor/SlotTest.java
@@ -26,6 +26,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.storm.daemon.supervisor.Slot.StaticState;
@@ -40,13 +41,20 @@ import org.apache.storm.generated.ProfileAction;
import org.apache.storm.generated.ProfileRequest;
import org.apache.storm.generated.WorkerResources;
import org.apache.storm.localizer.AsyncLocalizer;
+import org.apache.storm.localizer.BlobChangingCallback;
+import org.apache.storm.localizer.GoodToGo;
+import org.apache.storm.localizer.LocallyCachedBlob;
import org.apache.storm.scheduler.ISupervisor;
import org.apache.storm.utils.LocalState;
import org.apache.storm.utils.Time;
import org.apache.storm.utils.Time.SimulatedTime;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class SlotTest {
+ private static final Logger LOG = LoggerFactory.getLogger(SlotTest.class);
+
static WorkerResources mkWorkerResources(Double cpu, Double mem_on_heap, Double mem_off_heap) {
WorkerResources resources = new WorkerResources();
if (cpu != null) {
@@ -111,16 +119,36 @@ public class SlotTest {
assertFalse(Slot.equivalent(null, b));
assertFalse(Slot.equivalent(a, b));
}
-
+
+ @Test
+ public void testForSameTopology() {
+ LocalAssignment a = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 100.0, 100.0));
+ LocalAssignment aResized = mkLocalAssignment("A", mkExecutorInfoList(1,2,3,4,5), mkWorkerResources(100.0, 200.0, 100.0));
+ LocalAssignment b = mkLocalAssignment("B", mkExecutorInfoList(1,2,3,4,5,6), mkWorkerResources(100.0, 100.0, 100.0));
+ LocalAssignment bReordered = mkLocalAssignment("B", mkExecutorInfoList(6,5,4,3,2,1), mkWorkerResources(100.0, 100.0, 100.0));
+
+ assertTrue(Slot.forSameTopology(null, null));
+ assertTrue(Slot.forSameTopology(a, a));
+ assertTrue(Slot.forSameTopology(a, aResized));
+ assertTrue(Slot.forSameTopology(aResized, a));
+ assertTrue(Slot.forSameTopology(b, bReordered));
+ assertTrue(Slot.forSameTopology(bReordered, b));
+
+ assertFalse(Slot.forSameTopology(a, null));
+ assertFalse(Slot.forSameTopology(null, b));
+ assertFalse(Slot.forSameTopology(a, b));
+ }
+
@Test
public void testEmptyToEmpty() throws Exception {
try (SimulatedTime t = new SimulatedTime(1010)){
AsyncLocalizer localizer = mock(AsyncLocalizer.class);
LocalState state = mock(LocalState.class);
+ BlobChangingCallback cb = mock(BlobChangingCallback.class);
ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
ISupervisor iSuper = mock(ISupervisor.class);
StaticState staticState = new StaticState(localizer, 1000, 1000, 1000, 1000,
- containerLauncher, "localhost", 8080, iSuper, state);
+ containerLauncher, "localhost", 8080, iSuper, state, cb);
DynamicState dynamicState = new DynamicState(null, null, null);
DynamicState nextState = Slot.handleEmpty(dynamicState, staticState);
assertEquals(MachineState.EMPTY, nextState.state);
@@ -136,39 +164,28 @@ public class SlotTest {
List<ExecutorInfo> execList = mkExecutorInfoList(1,2,3,4,5);
LocalAssignment newAssignment =
mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
-
+
AsyncLocalizer localizer = mock(AsyncLocalizer.class);
+ BlobChangingCallback cb = mock(BlobChangingCallback.class);
Container container = mock(Container.class);
LocalState state = mock(LocalState.class);
ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
when(containerLauncher.launchContainer(port, newAssignment, state)).thenReturn(container);
LSWorkerHeartbeat hb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs());
when(container.readHeartbeat()).thenReturn(hb, hb);
-
- @SuppressWarnings("unchecked")
- CompletableFuture<Void> baseFuture = mock(CompletableFuture.class);
- when(localizer.requestDownloadBaseTopologyBlobs(newAssignment, port)).thenReturn(baseFuture);
-
+
@SuppressWarnings("unchecked")
CompletableFuture<Void> blobFuture = mock(CompletableFuture.class);
- when(localizer.requestDownloadTopologyBlobs(newAssignment, port)).thenReturn(blobFuture);
+ when(localizer.requestDownloadTopologyBlobs(newAssignment, port, cb)).thenReturn(blobFuture);
ISupervisor iSuper = mock(ISupervisor.class);
StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
- containerLauncher, "localhost", port, iSuper, state);
+ containerLauncher, "localhost", port, iSuper, state, cb);
DynamicState dynamicState = new DynamicState(null, null, null)
.withNewAssignment(newAssignment);
-
+
DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
- verify(localizer).requestDownloadBaseTopologyBlobs(newAssignment, port);
- assertEquals(MachineState.WAITING_FOR_BASIC_LOCALIZATION, nextState.state);
- assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
- assertEquals(newAssignment, nextState.pendingLocalization);
- assertEquals(0, Time.currentTimeMillis());
-
- nextState = Slot.stateMachineStep(nextState, staticState);
- verify(baseFuture).get(1000, TimeUnit.MILLISECONDS);
- verify(localizer).requestDownloadTopologyBlobs(newAssignment, port);
+ verify(localizer).requestDownloadTopologyBlobs(newAssignment, port, cb);
assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
assertEquals(newAssignment, nextState.pendingLocalization);
@@ -221,6 +238,7 @@ public class SlotTest {
mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
AsyncLocalizer localizer = mock(AsyncLocalizer.class);
+ BlobChangingCallback cb = mock(BlobChangingCallback.class);
Container container = mock(Container.class);
ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
LSWorkerHeartbeat oldhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs()-10);
@@ -231,7 +249,7 @@ public class SlotTest {
ISupervisor iSuper = mock(ISupervisor.class);
LocalState state = mock(LocalState.class);
StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
- containerLauncher, "localhost", port, iSuper, state);
+ containerLauncher, "localhost", port, iSuper, state, cb);
DynamicState dynamicState = new DynamicState(assignment, container, assignment);
DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
@@ -265,7 +283,9 @@ public class SlotTest {
List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
LocalAssignment cAssignment =
mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
-
+
+ BlobChangingCallback cb = mock(BlobChangingCallback.class);
+
Container cContainer = mock(Container.class);
LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
when(cContainer.readHeartbeat()).thenReturn(chb);
@@ -285,48 +305,36 @@ public class SlotTest {
when(nContainer.readHeartbeat()).thenReturn(nhb, nhb);
@SuppressWarnings("unchecked")
- CompletableFuture<Void> baseFuture = mock(CompletableFuture.class);
- when(localizer.requestDownloadBaseTopologyBlobs(nAssignment, port)).thenReturn(baseFuture);
-
- @SuppressWarnings("unchecked")
CompletableFuture<Void> blobFuture = mock(CompletableFuture.class);
- when(localizer.requestDownloadTopologyBlobs(nAssignment, port)).thenReturn(blobFuture);
+ when(localizer.requestDownloadTopologyBlobs(nAssignment, port, cb)).thenReturn(blobFuture);
ISupervisor iSuper = mock(ISupervisor.class);
StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
- containerLauncher, "localhost", port, iSuper, state);
+ containerLauncher, "localhost", port, iSuper, state, cb);
DynamicState dynamicState = new DynamicState(cAssignment, cContainer, nAssignment);
DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
assertEquals(MachineState.KILL, nextState.state);
verify(cContainer).kill();
- verify(localizer).requestDownloadBaseTopologyBlobs(nAssignment, port);
- assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
+ verify(localizer).requestDownloadTopologyBlobs(nAssignment, port, cb);
+ assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
assertEquals(nAssignment, nextState.pendingLocalization);
assertTrue(Time.currentTimeMillis() > 1000);
nextState = Slot.stateMachineStep(nextState, staticState);
assertEquals(MachineState.KILL, nextState.state);
verify(cContainer).forceKill();
- assertSame("pendingDownload not set properly", baseFuture, nextState.pendingDownload);
+ assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
assertEquals(nAssignment, nextState.pendingLocalization);
assertTrue(Time.currentTimeMillis() > 2000);
nextState = Slot.stateMachineStep(nextState, staticState);
- assertEquals(MachineState.WAITING_FOR_BASIC_LOCALIZATION, nextState.state);
+ assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
verify(cContainer).cleanUp();
verify(localizer).releaseSlotFor(cAssignment, port);
assertTrue(Time.currentTimeMillis() > 2000);
nextState = Slot.stateMachineStep(nextState, staticState);
- assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
- verify(baseFuture).get(1000, TimeUnit.MILLISECONDS);
- verify(localizer).requestDownloadTopologyBlobs(nAssignment, port);
- assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
- assertEquals(nAssignment, nextState.pendingLocalization);
- assertTrue(Time.currentTimeMillis() > 2000);
-
- nextState = Slot.stateMachineStep(nextState, staticState);
verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
verify(containerLauncher).launchContainer(port, nAssignment, state);
assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
@@ -361,7 +369,6 @@ public class SlotTest {
assertTrue(Time.currentTimeMillis() > 4000);
}
}
-
@Test
public void testRunningToEmpty() throws Exception {
@@ -378,18 +385,19 @@ public class SlotTest {
when(cContainer.areAllProcessesDead()).thenReturn(false, true);
AsyncLocalizer localizer = mock(AsyncLocalizer.class);
+ BlobChangingCallback cb = mock(BlobChangingCallback.class);
ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
ISupervisor iSuper = mock(ISupervisor.class);
LocalState state = mock(LocalState.class);
StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
- containerLauncher, "localhost", port, iSuper, state);
+ containerLauncher, "localhost", port, iSuper, state, cb);
DynamicState dynamicState = new DynamicState(cAssignment, cContainer, null);
DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
assertEquals(MachineState.KILL, nextState.state);
verify(cContainer).kill();
- verify(localizer, never()).requestDownloadBaseTopologyBlobs(null, port);
+ verify(localizer, never()).requestDownloadTopologyBlobs(null, port, cb);
assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
assertEquals(null, nextState.pendingLocalization);
assertTrue(Time.currentTimeMillis() > 1000);
@@ -438,12 +446,13 @@ public class SlotTest {
when(cContainer.runProfiling(any(ProfileRequest.class), anyBoolean())).thenReturn(true);
AsyncLocalizer localizer = mock(AsyncLocalizer.class);
+ BlobChangingCallback cb = mock(BlobChangingCallback.class);
ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
ISupervisor iSuper = mock(ISupervisor.class);
LocalState state = mock(LocalState.class);
StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
- containerLauncher, "localhost", port, iSuper, state);
+ containerLauncher, "localhost", port, iSuper, state, cb);
Set<TopoProfileAction> profileActions = new HashSet<>();
ProfileRequest request = new ProfileRequest();
request.set_action(ProfileAction.JPROFILE_STOP);
@@ -495,4 +504,103 @@ public class SlotTest {
assertTrue(Time.currentTimeMillis() > 5000);
}
}
+
+ @Test
+ public void testResourcesChanged() throws Exception {
+ try (SimulatedTime t = new SimulatedTime(1010)){
+ int port = 8080;
+ String cTopoId = "CURRENT";
+ List<ExecutorInfo> cExecList = mkExecutorInfoList(1,2,3,4,5);
+ LocalAssignment cAssignment =
+ mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
+
+ BlobChangingCallback cb = mock(BlobChangingCallback.class);
+
+ Container cContainer = mock(Container.class);
+ LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
+ when(cContainer.readHeartbeat()).thenReturn(chb);
+ when(cContainer.areAllProcessesDead()).thenReturn(false, true);
+
+ AsyncLocalizer localizer = mock(AsyncLocalizer.class);
+ Container nContainer = mock(Container.class);
+ LocalState state = mock(LocalState.class);
+ ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
+ when(containerLauncher.launchContainer(port, cAssignment, state)).thenReturn(nContainer);
+ when(nContainer.readHeartbeat()).thenReturn(chb, chb);
+
+ ISupervisor iSuper = mock(ISupervisor.class);
+ StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000,
+ containerLauncher, "localhost", port, iSuper, state, cb);
+
+ Set<Slot.BlobChangeing> changing = new HashSet<>();
+ LocallyCachedBlob stormJar = mock(LocallyCachedBlob.class);
+ GoodToGo.GoodToGoLatch stormJarLatch = mock(GoodToGo.GoodToGoLatch.class);
+ CompletableFuture<Void> stormJarLatchFuture = mock(CompletableFuture.class);
+ when(stormJarLatch.countDown()).thenReturn(stormJarLatchFuture);
+ changing.add(new Slot.BlobChangeing(cAssignment, stormJar, stormJarLatch));
+
+ DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment).withChangingBlobs(changing);
+
+ DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
+ assertEquals(MachineState.KILL_BLOB_UPDATE, nextState.state);
+ verify(iSuper).killedWorker(port);
+ verify(cContainer).kill();
+ verify(localizer, never()).requestDownloadTopologyBlobs(any(), anyInt(), any());
+ assertNull(nextState.pendingDownload);
+ assertNull(nextState.pendingLocalization);
+ assertEquals(changing, nextState.changingBlobs);
+ assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+ assertNull(nextState.pendingChangingBlobsAssignment);
+ assertTrue(Time.currentTimeMillis() > 1000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.KILL_BLOB_UPDATE, nextState.state);
+ verify(cContainer).forceKill();
+ assertNull(nextState.pendingDownload);
+ assertNull(nextState.pendingLocalization);
+ assertEquals(changing, nextState.changingBlobs);
+ assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+ assertNull(nextState.pendingChangingBlobsAssignment);
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.WAITING_FOR_BLOB_UPDATE, nextState.state);
+ verify(cContainer).cleanUp();
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ verify(stormJarLatchFuture).get(anyLong(), any());
+ verify(containerLauncher).launchContainer(port, cAssignment, state);
+ assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
+ assertNull(nextState.pendingChangingBlobsAssignment);
+ assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+ assertSame(cAssignment, nextState.currentAssignment);
+ assertSame(nContainer, nextState.container);
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertNull(nextState.pendingChangingBlobsAssignment);
+ assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+ assertSame(cAssignment, nextState.currentAssignment);
+ assertSame(nContainer, nextState.container);
+ assertTrue(Time.currentTimeMillis() > 2000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertNull(nextState.pendingChangingBlobsAssignment);
+ assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+ assertSame(cAssignment, nextState.currentAssignment);
+ assertSame(nContainer, nextState.container);
+ assertTrue(Time.currentTimeMillis() > 3000);
+
+ nextState = Slot.stateMachineStep(nextState, staticState);
+ assertEquals(MachineState.RUNNING, nextState.state);
+ assertNull(nextState.pendingChangingBlobsAssignment);
+ assertTrue(nextState.pendingChangeingBlobs.isEmpty());
+ assertSame(cAssignment, nextState.currentAssignment);
+ assertSame(nContainer, nextState.container);
+ assertTrue(Time.currentTimeMillis() > 4000);
+ }
+ }
}