You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/09/03 23:09:19 UTC
[1/2] storm git commit: STORM-2660 The Nimbus storm-local directory
is relative to the working directory of the shell executing "storm nimbus"
Repository: storm
Updated Branches:
refs/heads/master 758a1d2d6 -> aefc72520
STORM-2660 The Nimbus storm-local directory is relative to the working directory of the shell executing "storm nimbus"
* Let Nimbus uses same storm-local directory as Supervisor which is known to be valid
* also addresses other places as well (review comment from @srdo)
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/96a08665
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/96a08665
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/96a08665
Branch: refs/heads/master
Commit: 96a08665ca5cceda541514872e904079e59b486a
Parents: 758a1d2
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Thu Aug 31 18:26:59 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Sep 4 08:08:37 2017 +0900
----------------------------------------------------------------------
storm-client/src/jvm/org/apache/storm/Config.java | 6 ++++--
.../src/jvm/org/apache/storm/utils/ConfigUtils.java | 14 ++++++++++++++
.../org/apache/storm/blobstore/LocalFsBlobStore.java | 7 ++-----
.../scheduler/utils/ArtifactoryConfigLoader.java | 14 +++++---------
.../org/apache/storm/utils/ServerConfigUtils.java | 2 +-
5 files changed, 26 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/96a08665/storm-client/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/Config.java b/storm-client/src/jvm/org/apache/storm/Config.java
index 2965b79..13b8a22 100644
--- a/storm-client/src/jvm/org/apache/storm/Config.java
+++ b/storm-client/src/jvm/org/apache/storm/Config.java
@@ -1031,7 +1031,8 @@ public class Config extends HashMap<String, Object> {
/**
* What directory to use for the blobstore. The directory is expected to be an
* absolute path when using HDFS blobstore, for LocalFsBlobStore it could be either
- * absolute or relative.
+ * absolute or relative. If the setting is a relative directory, it is relative to
+ * root directory of Storm installation.
*/
@isString
public static final String BLOBSTORE_DIR = "blobstore.dir";
@@ -1424,7 +1425,8 @@ public class Config extends HashMap<String, Object> {
/**
* A directory on the local filesystem used by Storm for any local
* filesystem usage it needs. The directory must exist and the Storm daemons must
- * have permission to read/write from this location.
+ * have permission to read/write from this location. It could be either absolute or relative.
+ * If the setting is a relative directory, it is relative to root directory of Storm installation.
*/
@isString
public static final String STORM_LOCAL_DIR = "storm.local.dir";
http://git-wip-us.apache.org/repos/asf/storm/blob/96a08665/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java b/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
index 6ea79dd..4a54ac7 100644
--- a/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
+++ b/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
@@ -215,6 +215,20 @@ public class ConfigUtils {
}
}
+ public static String absoluteStormBlobStoreDir(Map<String, Object> conf) {
+ String stormHome = System.getProperty("storm.home");
+ String blobStoreDir = (String) conf.get(Config.BLOBSTORE_DIR);
+ if (blobStoreDir == null) {
+ return ConfigUtils.absoluteStormLocalDir(conf);
+ } else {
+ if (new File(blobStoreDir).isAbsolute()) {
+ return blobStoreDir;
+ } else {
+ return (stormHome + FILE_SEPARATOR + blobStoreDir);
+ }
+ }
+ }
+
public static StormTopology readSupervisorStormCodeGivenPath(String stormCodePath, AdvancedFSOps ops) throws IOException {
return Utils.deserialize(ops.slurp(new File(stormCodePath)), StormTopology.class);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/96a08665/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
index 87c8dce..2f73334 100644
--- a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
+++ b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
@@ -17,7 +17,6 @@
*/
package org.apache.storm.blobstore;
-import org.apache.storm.Config;
import org.apache.storm.generated.SettableBlobMeta;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.KeyAlreadyExistsException;
@@ -25,6 +24,7 @@ import org.apache.storm.generated.KeyNotFoundException;
import org.apache.storm.generated.ReadableBlobMeta;
import org.apache.storm.nimbus.NimbusInfo;
+import org.apache.storm.utils.ConfigUtils;
import org.apache.storm.utils.Utils;
import org.apache.zookeeper.KeeperException.NoNodeException;
import org.apache.curator.framework.CuratorFramework;
@@ -85,10 +85,7 @@ public class LocalFsBlobStore extends BlobStore {
this.nimbusInfo = nimbusInfo;
zkClient = BlobStoreUtils.createZKClient(conf);
if (overrideBase == null) {
- overrideBase = (String)conf.get(Config.BLOBSTORE_DIR);
- if (overrideBase == null) {
- overrideBase = (String) conf.get(Config.STORM_LOCAL_DIR);
- }
+ overrideBase = ConfigUtils.absoluteStormBlobStoreDir(conf);
}
File baseDir = new File(overrideBase, BASE_BLOBS_DIR_NAME);
try {
http://git-wip-us.apache.org/repos/asf/storm/blob/96a08665/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
index 3afa903..87f39cd 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
@@ -37,6 +37,8 @@ import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.apache.storm.Config;
import org.apache.storm.DaemonConfig;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ServerConfigUtils;
import org.apache.storm.utils.Time;
import org.apache.storm.utils.Utils;
import org.json.simple.JSONArray;
@@ -330,15 +332,9 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
}
}
- private void makeArtifactoryCache(String location) {
- // Just make sure appropriate directories exist
- String localDirName = (String) conf.get(Config.STORM_LOCAL_DIR);
- if (localDirName == null) {
- return;
- }
-
+ private void makeArtifactoryCache(String location) throws IOException {
// First make the cache dir
- localDirName = localDirName + File.separator + "nimbus" + File.separator + LOCAL_ARTIFACT_DIR;
+ String localDirName = ServerConfigUtils.masterLocalDir(conf) + File.separator + LOCAL_ARTIFACT_DIR;
File dir = new File(localDirName);
if (! dir.exists()) {
dir.mkdirs();
@@ -352,7 +348,7 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
cacheInitialized = true;
}
- private Map loadFromURI(URI uri) {
+ private Map loadFromURI(URI uri) throws IOException {
String host = uri.getHost();
Integer port = uri.getPort();
String location = uri.getPath();
http://git-wip-us.apache.org/repos/asf/storm/blob/96a08665/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java b/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
index 426e190..480d523 100644
--- a/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
@@ -55,7 +55,7 @@ public class ServerConfigUtils {
}
public static String masterLocalDir(Map<String, Object> conf) throws IOException {
- String ret = String.valueOf(conf.get(Config.STORM_LOCAL_DIR)) + FILE_SEPARATOR + "nimbus";
+ String ret = ConfigUtils.absoluteStormLocalDir(conf) + FILE_SEPARATOR + "nimbus";
FileUtils.forceMkdir(new File(ret));
return ret;
}
[2/2] storm git commit: Merge branch 'STORM-2660'
Posted by ka...@apache.org.
Merge branch 'STORM-2660'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/aefc7252
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/aefc7252
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/aefc7252
Branch: refs/heads/master
Commit: aefc725201a037745fba7c95e1ac34ffb266207f
Parents: 758a1d2 96a0866
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Mon Sep 4 08:09:13 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Sep 4 08:09:13 2017 +0900
----------------------------------------------------------------------
storm-client/src/jvm/org/apache/storm/Config.java | 6 ++++--
.../src/jvm/org/apache/storm/utils/ConfigUtils.java | 14 ++++++++++++++
.../org/apache/storm/blobstore/LocalFsBlobStore.java | 7 ++-----
.../scheduler/utils/ArtifactoryConfigLoader.java | 14 +++++---------
.../org/apache/storm/utils/ServerConfigUtils.java | 2 +-
5 files changed, 26 insertions(+), 17 deletions(-)
----------------------------------------------------------------------