You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2016/09/25 12:09:12 UTC
[1/3] storm git commit: STORM-2122: Cache dependency data,
and serialize reading of the data.
Repository: storm
Updated Branches:
refs/heads/master 5fea60211 -> 22e9609e9
STORM-2122: Cache dependency data, and serialize reading of the data.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e55d9b7d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e55d9b7d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e55d9b7d
Branch: refs/heads/master
Commit: e55d9b7d47a8ea62174be106af0f9ab1cea402eb
Parents: d2ca97f
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Sep 23 11:26:34 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Sep 23 11:26:34 2016 -0500
----------------------------------------------------------------------
.../storm/daemon/supervisor/BasicContainer.java | 93 +++++++++++++++++---
1 file changed, 79 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e55d9b7d/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
index 93c10c7..b89d0f1 100644
--- a/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
+++ b/storm-core/src/jvm/org/apache/storm/daemon/supervisor/BasicContainer.java
@@ -23,8 +23,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.Iterator;
import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -470,6 +471,82 @@ public class BasicContainer extends Container {
return log4jConfigurationDir + Utils.FILE_PATH_SEPARATOR + "worker.xml";
}
+ private static class DependencyLocations {
+ private List<String> _data = null;
+ private final Map<String, Object> _conf;
+ private final String _topologyId;
+ private final AdvancedFSOps _ops;
+ private final String _stormRoot;
+
+ public DependencyLocations(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, final String stormRoot) {
+ _conf = conf;
+ _topologyId = topologyId;
+ _ops = ops;
+ _stormRoot = stormRoot;
+ }
+
+ public String toString() {
+ List<String> data;
+ synchronized(this) {
+ data = _data;
+ }
+ return "DEP_LOCS for " + _topologyId +" => " + data;
+ }
+
+ public synchronized List<String> get() throws IOException {
+ if (_data != null) {
+ return _data;
+ }
+ final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _ops);
+ final List<String> dependencyLocations = new ArrayList<>();
+ if (stormTopology.get_dependency_jars() != null) {
+ for (String dependency : stormTopology.get_dependency_jars()) {
+ dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath());
+ }
+ }
+
+ if (stormTopology.get_dependency_artifacts() != null) {
+ for (String dependency : stormTopology.get_dependency_artifacts()) {
+ dependencyLocations.add(new File(_stormRoot, dependency).getAbsolutePath());
+ }
+ }
+ _data = dependencyLocations;
+ return _data;
+ }
+ }
+
+ static class DepLRUCache {
+ public final int _maxSize = 100; //We could make this configurable in the future...
+
+ @SuppressWarnings("serial")
+ private LinkedHashMap<String, DependencyLocations> _cache = new LinkedHashMap<String, DependencyLocations>() {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<String,DependencyLocations> eldest) {
+ return (size() > _maxSize);
+ }
+ };
+
+ public synchronized DependencyLocations get(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, String stormRoot) {
+ //Only go off of the topology id for now.
+ DependencyLocations dl = _cache.get(topologyId);
+ if (dl == null) {
+ _cache.putIfAbsent(topologyId, new DependencyLocations(conf, topologyId, ops, stormRoot));
+ dl = _cache.get(topologyId);
+ }
+ return dl;
+ }
+
+ public synchronized void clear() {
+ _cache.clear();
+ }
+ }
+
+ static final DepLRUCache DEP_LOC_CACHE = new DepLRUCache();
+
+ public static List<String> getDependencyLocationsFor(final Map<String, Object> conf, final String topologyId, final AdvancedFSOps ops, String stormRoot) throws IOException {
+ return DEP_LOC_CACHE.get(conf, topologyId, ops, stormRoot).get();
+ }
+
/**
* Get parameters for the class path of the worker process. Also used by the
* log Writer
@@ -479,19 +556,7 @@ public class BasicContainer extends Container {
*/
private List<String> getClassPathParams(final String stormRoot) throws IOException {
final String stormJar = ConfigUtils.supervisorStormJarPath(stormRoot);
- final StormTopology stormTopology = ConfigUtils.readSupervisorTopology(_conf, _topologyId, _ops);
- final List<String> dependencyLocations = new ArrayList<>();
- if (stormTopology.get_dependency_jars() != null) {
- for (String dependency : stormTopology.get_dependency_jars()) {
- dependencyLocations.add(new File(stormRoot, dependency).getAbsolutePath());
- }
- }
-
- if (stormTopology.get_dependency_artifacts() != null) {
- for (String dependency : stormTopology.get_dependency_artifacts()) {
- dependencyLocations.add(new File(stormRoot, dependency).getAbsolutePath());
- }
- }
+ final List<String> dependencyLocations = getDependencyLocationsFor(_conf, _topologyId, _ops, stormRoot);
final String workerClassPath = getWorkerClassPath(stormJar, dependencyLocations);
List<String> classPathParams = new ArrayList<>();
[3/3] storm git commit: add STORM-2122 to CHANGELOG
Posted by ka...@apache.org.
add STORM-2122 to CHANGELOG
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/22e9609e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/22e9609e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/22e9609e
Branch: refs/heads/master
Commit: 22e9609e933587436aee018009684d5599c0f47b
Parents: 1056efb
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Sun Sep 25 21:08:56 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Sun Sep 25 21:08:56 2016 +0900
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/22e9609e/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1a60243..eed694d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 2.0.0
+ * STORM-2122: Cache dependency data, and serialize reading of the data
* STORM-2117: Supervisor V2 with local mode extracts resources directory to the wrong directory
* STORM-2110: strip out empty String in worker opts
* STORM-2100: Fix Trident SQL join tests to not rely on ordering
[2/3] storm git commit: Merge branch 'STORM-2122' of
https://github.com/revans2/incubator-storm into STORM-2122
Posted by ka...@apache.org.
Merge branch 'STORM-2122' of https://github.com/revans2/incubator-storm into STORM-2122
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/1056efb2
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/1056efb2
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/1056efb2
Branch: refs/heads/master
Commit: 1056efb227409c136e715da0f93a06633b53af8c
Parents: 5fea602 e55d9b7
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Sun Sep 25 20:57:04 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Sun Sep 25 20:57:04 2016 +0900
----------------------------------------------------------------------
.../storm/daemon/supervisor/BasicContainer.java | 93 +++++++++++++++++---
1 file changed, 79 insertions(+), 14 deletions(-)
----------------------------------------------------------------------