You are viewing a plain text version of this content. The canonical link for it is here.
Posted to s4-commits@incubator.apache.org by mm...@apache.org on 2013/01/23 22:35:05 UTC
[5/7] git commit: S4-59 fixed s4-tools commands with new
configuration scheme - also fixed some parameters names that needed an update
+ doc
S4-59 fixed s4-tools commands with new configuration scheme
- also fixed some parameters names that needed an update + doc
Project: http://git-wip-us.apache.org/repos/asf/incubator-s4/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s4/commit/7548ba53
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s4/tree/7548ba53
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s4/diff/7548ba53
Branch: refs/heads/dev
Commit: 7548ba53b70e9131dd30e437e392d2c0b6e30731
Parents: a941dfd
Author: Matthieu Morel <mm...@apache.org>
Authored: Tue Jan 22 17:42:46 2013 +0100
Committer: Matthieu Morel <mm...@apache.org>
Committed: Tue Jan 22 17:49:48 2013 +0100
----------------------------------------------------------------------
.../main/java/org/apache/s4/core/S4Bootstrap.java | 4 +-
.../src/main/java/org/apache/s4/core/S4Node.java | 4 +-
.../java/org/apache/s4/core/util/AppConfig.java | 12 ++++++--
.../s4/deploy/prodcon/TestProducerConsumer.java | 2 +-
.../src/main/java/org/apache/s4/tools/Deploy.java | 23 ++++++++++++--
.../src/main/java/org/apache/s4/tools/Status.java | 14 +++++----
.../main/java/org/apache/s4/tools/ZKServer.java | 4 +-
.../src/main/resources/templates/newApp.README | 7 +++-
.../s4-tools/src/main/resources/templates/s4 | 8 ++--
test-apps/twitter-adapter/build.gradle | 2 +-
10 files changed, 53 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Bootstrap.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Bootstrap.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Bootstrap.java
index deb14ad..e469121 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Bootstrap.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Bootstrap.java
@@ -107,7 +107,7 @@ public class S4Bootstrap {
// can be null
final AppConfig appConfig = new AppConfig(appData);
- String appName = appData.getSimpleField("name");
+ String appName = appConfig.getAppName();
List<File> modulesLocalCopies = new ArrayList<File>();
@@ -120,7 +120,7 @@ public class S4Bootstrap {
@Override
public void run() {
- // load Main class through modules classloader and start it
+ // load app class through modules classloader and start it
S4Bootstrap.startS4App(appConfig, parentInjector, modulesLoader);
signalOneAppLoaded.countDown();
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
index 4bd929b..907fcb9 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
@@ -39,7 +39,7 @@ public class S4Node {
}
- private static void startNode(S4NodeArgs mainArgs) throws InterruptedException, IOException {
+ private static void startNode(S4NodeArgs nodeArgs) throws InterruptedException, IOException {
Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler() {
@Override
@@ -50,7 +50,7 @@ public class S4Node {
});
Injector injector = Guice.createInjector(new Module[] { new BaseModule(Resources.getResource(
- "default.s4.base.properties").openStream(), mainArgs.clusterName) });
+ "default.s4.base.properties").openStream(), nodeArgs.clusterName) });
S4Bootstrap bootstrap = injector.getInstance(S4Bootstrap.class);
try {
bootstrap.start(injector);
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-core/src/main/java/org/apache/s4/core/util/AppConfig.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/util/AppConfig.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/AppConfig.java
index 9bd09c7..a31ff17 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/util/AppConfig.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/AppConfig.java
@@ -137,17 +137,23 @@ public class AppConfig {
}
public Builder customModulesNames(List<String> customModulesNames) {
- config.customModulesNames = customModulesNames;
+ if (customModulesNames != null) {
+ config.customModulesNames = customModulesNames;
+ }
return this;
}
public Builder customModulesURIs(List<String> customModulesURIs) {
- config.customModulesURIs = customModulesURIs;
+ if (customModulesURIs != null) {
+ config.customModulesURIs = customModulesURIs;
+ }
return this;
}
public Builder namedParameters(Map<String, String> namedParameters) {
- config.namedParameters = namedParameters;
+ if (namedParameters != null) {
+ config.namedParameters = namedParameters;
+ }
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-core/src/test/java/org/apache/s4/deploy/prodcon/TestProducerConsumer.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/deploy/prodcon/TestProducerConsumer.java b/subprojects/s4-core/src/test/java/org/apache/s4/deploy/prodcon/TestProducerConsumer.java
index b48d371..31c6a34 100644
--- a/subprojects/s4-core/src/test/java/org/apache/s4/deploy/prodcon/TestProducerConsumer.java
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/deploy/prodcon/TestProducerConsumer.java
@@ -147,7 +147,7 @@ public class TestProducerConsumer {
"localhost:2181");
// that may be a bit long to complete...
- Assert.assertTrue(signalConsumptionComplete.await(30, TimeUnit.SECONDS));
+ Assert.assertTrue(signalConsumptionComplete.await(40, TimeUnit.SECONDS));
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
index e39d00f..69ebaa6 100644
--- a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
+++ b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
@@ -70,9 +70,10 @@ public class Deploy extends S4ArgsBase {
System.exit(1);
}
- File s4rToDeploy;
+ File s4rToDeploy = null;
if (deployArgs.s4rPath != null) {
+ // 1. if there is an application archive, we use it
s4rToDeploy = new File(deployArgs.s4rPath);
if (!s4rToDeploy.exists()) {
logger.error("Specified S4R file does not exist in {}", s4rToDeploy.getAbsolutePath());
@@ -82,7 +83,10 @@ public class Deploy extends S4ArgsBase {
"Using specified S4R [{}], the S4R archive will not be built from source (and corresponding parameters are ignored)",
s4rToDeploy.getAbsolutePath());
}
- } else {
+ } else if (deployArgs.gradleBuildFile != null) {
+
+ // 2. otherwise if there is a build file, we create the S4R archive from that
+
List<String> params = new ArrayList<String>();
// prepare gradle -P parameters, including passed gradle opts
params.addAll(deployArgs.gradleOpts);
@@ -103,11 +107,22 @@ public class Deploy extends S4ArgsBase {
} else {
s4rToDeploy = tmpS4R;
}
+ } else {
+ if (!Strings.isNullOrEmpty(deployArgs.appClass)) {
+ // 3. otherwise if there is at least an app class specified (e.g. for running "s4 adapter"), we use
+ // it and won't use an S4R
+ logger.info("No S4R path specified, nor build file specified: this assumes the app is in the classpath");
+ } else {
+ logger.error("You must specify an S4R file, a build file to create an S4R from, or an appClass that will be in the classpath");
+ System.exit(1);
+ }
+
}
DeploymentUtils.initAppConfig(
- new AppConfig.Builder().appName(deployArgs.appName).appURI(s4rToDeploy.toURI().toString())
- .customModulesNames(deployArgs.modulesClassesNames)
+ new AppConfig.Builder().appName(deployArgs.appName)
+ .appURI(s4rToDeploy != null ? s4rToDeploy.toURI().toString() : null)
+ .appClassName(deployArgs.appClass).customModulesNames(deployArgs.modulesClassesNames)
.customModulesURIs(deployArgs.modulesURIs)
.namedParameters(convertListArgsToMap(deployArgs.extraNamedParameters)).build(),
deployArgs.clusterName, false, deployArgs.zkConnectionString);
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Status.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Status.java b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Status.java
index 824aa03..09ae37d 100644
--- a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Status.java
+++ b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Status.java
@@ -30,7 +30,7 @@ import org.apache.s4.comm.topology.ClusterNode;
import org.apache.s4.comm.topology.ZNRecord;
import org.apache.s4.comm.topology.ZNRecordSerializer;
import org.apache.s4.comm.topology.ZkClient;
-import org.apache.s4.tools.S4ArgsBase.GradleOptsConverter;
+import org.apache.s4.core.util.AppConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -219,7 +219,7 @@ public class Status extends S4ArgsBase {
/**
* show as cluster1(app1), cluster2(app2)
- *
+ *
* @param clusters
* cluster list
* @param clusterAppMap
@@ -298,8 +298,9 @@ public class Status extends S4ArgsBase {
private static String getApp(String clusterName, ZkClient zkClient) {
String appPath = "/s4/clusters/" + clusterName + "/app/s4App";
if (zkClient.exists(appPath)) {
- ZNRecord appRecord = zkClient.readData("/s4/clusters/" + clusterName + "/app/s4App");
- return appRecord.getSimpleField("name");
+ AppConfig appConfig = new AppConfig((ZNRecord) zkClient.readData("/s4/clusters/" + clusterName
+ + "/app/s4App"));
+ return appConfig.getAppName();
}
return NONE;
}
@@ -357,8 +358,9 @@ public class Status extends S4ArgsBase {
app.cluster = clusterName;
try {
ZNRecord appRecord = zkClient.readData(appPath);
- app.name = appRecord.getSimpleField("name");
- app.uri = appRecord.getSimpleField("s4r_uri");
+ AppConfig appConfig = new AppConfig(appRecord);
+ app.name = appConfig.getAppName();
+ app.uri = appConfig.getAppURI();
} catch (ZkNoNodeException e) {
logger.warn(appPath + " doesn't exist");
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-tools/src/main/java/org/apache/s4/tools/ZKServer.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/ZKServer.java b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/ZKServer.java
index 4893928..580872b 100644
--- a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/ZKServer.java
+++ b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/ZKServer.java
@@ -107,8 +107,8 @@ public class ZKServer {
String dataDir = new File(System.getProperty("java.io.tmpdir") + File.separator + "tmp" + File.separator
+ "zookeeper" + File.separator + "data").getAbsolutePath();
- @Parameter(names = "-clean", description = "clean zookeeper data (arity=0) (make sure you specify correct directories...)")
- boolean clean = false;
+ @Parameter(names = "-clean", description = "clean zookeeper data (make sure you specify correct directories...)")
+ boolean clean = true;
@Parameter(names = "-logDir", description = "log directory")
String logDir = new File(System.getProperty("java.io.tmpdir") + File.separator + "tmp" + File.separator
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-tools/src/main/resources/templates/newApp.README
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/resources/templates/newApp.README b/subprojects/s4-tools/src/main/resources/templates/newApp.README
index 052e303..e572737 100644
--- a/subprojects/s4-tools/src/main/resources/templates/newApp.README
+++ b/subprojects/s4-tools/src/main/resources/templates/newApp.README
@@ -32,6 +32,9 @@
- application 2 must define an output stream with the same name (stream1)
If you want to use a simple adapter process, listening to an external source, converting incoming data into S4 events, and sending that to S4 apps, you can define
- your own app that extends the AdapterApp class. Then to start it (for instance):
- - ./s4 adapter -cluster=c1 -appClass=org.apache.s4.example.twitter.TwitterInputAdapter -p=s4.adapter.output.stream=stream1
+ your own app that extends the AdapterApp class, for instance:
+ ./s4 deploy -appClass=hello.HelloInputAdapter -p=s4.adapter.output.stream=names -cluster=c2 -appName=adapter
+
+ Then in order to start a node and automatically using the classpath of the current project, use the "s4 adapter" command:
+ - ./s4 adapter -cluster=c2
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/subprojects/s4-tools/src/main/resources/templates/s4
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/resources/templates/s4 b/subprojects/s4-tools/src/main/resources/templates/s4
index 45c5efd..69e5c78 100644
--- a/subprojects/s4-tools/src/main/resources/templates/s4
+++ b/subprojects/s4-tools/src/main/resources/templates/s4
@@ -3,12 +3,12 @@
case "$1" in
"adapter")
# we need something different in order to pass the classpath of the current project
- # you must specify: appClassName (extends AdapterApp class) , cluster name, output stream name
- # current syntax is not yet straightforward:
- # example : ./s4 adapter -cluster=c1 -appClass=org.apache.s4.example.twitter.TwitterInputAdapter -p=s4.adapter.output.stream:s1
+ # you must specify: cluster name (and -zk param if different from localhost:2181)
+
+ # example : ./s4 adapter -cluster=c1
shift 1
<s4_install_dir>/gradlew cp
- java -cp `cat classpath.txt` org.apache.s4.core.Main $@
+ java -cp `cat classpath.txt` org.apache.s4.core.S4Node $@
;;
*)
echo "calling referenced s4 script : <s4_script_path>"
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/7548ba53/test-apps/twitter-adapter/build.gradle
----------------------------------------------------------------------
diff --git a/test-apps/twitter-adapter/build.gradle b/test-apps/twitter-adapter/build.gradle
index 9b6a2d4..9272fa1 100644
--- a/test-apps/twitter-adapter/build.gradle
+++ b/test-apps/twitter-adapter/build.gradle
@@ -55,7 +55,7 @@ apply plugin: 'eclipse'
apply plugin: 'idea'
apply plugin:'application'
-mainClassName = "org.apache.s4.core.Main"
+mainClassName = "org.apache.s4.core.S4Node"
/* The app classname is set automatically from the source files. */
def appClassname = ''