You are viewing a plain text version of this content. The canonical link for it is here.
Posted to s4-commits@incubator.apache.org by mm...@apache.org on 2012/07/05 13:12:06 UTC
[2/2] git commit: Fixed -zk parameter handling and modified the
configuration for inline parameters - inline parameters now specified as :
-={param1=value1}, {param2=value2}
Fixed -zk parameter handling and modified the configuration for inline
parameters
- inline parameters now specified as :
-<namedStringParameters|p>={param1=value1},{param2=value2}
Project: http://git-wip-us.apache.org/repos/asf/incubator-s4/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s4/commit/22961068
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s4/tree/22961068
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s4/diff/22961068
Branch: refs/heads/piper
Commit: 229610681adf96d9a08772777d55cffb71b3620c
Parents: e80164b
Author: Matthieu Morel <mm...@apache.org>
Authored: Wed Jul 4 15:39:11 2012 +0200
Committer: Matthieu Morel <mm...@apache.org>
Committed: Wed Jul 4 15:39:11 2012 +0200
----------------------------------------------------------------------
.../src/main/java/org/apache/s4/core/Main.java | 58 +++++++++----
.../src/main/java/org/apache/s4/core/Module.java | 65 ---------------
.../src/main/resources/templates/newApp.README | 8 +-
3 files changed, 45 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/22961068/subprojects/s4-core/src/main/java/org/apache/s4/core/Main.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/Main.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/Main.java
index 139ad35..5f1375b 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/Main.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/Main.java
@@ -8,12 +8,15 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.s4.comm.DefaultCommModule;
import org.apache.s4.core.util.ParametersInjectionModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.beust.jcommander.IStringConverter;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@@ -21,6 +24,9 @@ import com.google.common.io.Resources;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.google.inject.util.Modules;
+import com.google.inject.util.Modules.OverriddenModuleBuilder;
/**
* Bootstrap class for S4. It creates an S4 node.
@@ -74,6 +80,12 @@ public class Main {
coreConfigString = mainArgs.coreConfigFilePath;
}
+ logger.info(
+ "Initializing S4 node with : \n- comm module class [{}]\n- comm configuration file [{}]\n- core module class [{}]\n- core configuration file[{}]\n- extra modules: {}\n- inline parameters: {}",
+ new String[] { mainArgs.commModuleClass, commConfigString, mainArgs.coreModuleClass,
+ coreConfigString, Arrays.toString(mainArgs.extraModulesClasses.toArray(new String[] {})),
+ Arrays.toString(mainArgs.extraNamedParameters.toArray(new String[] {})) });
+
AbstractModule commModule = (AbstractModule) Class.forName(mainArgs.commModuleClass)
.getConstructor(InputStream.class, String.class)
.newInstance(commConfigFileInputStream, mainArgs.clusterName);
@@ -82,31 +94,31 @@ public class Main {
List<com.google.inject.Module> extraModules = new ArrayList<com.google.inject.Module>();
for (String moduleClass : mainArgs.extraModulesClasses) {
- extraModules.add((com.google.inject.Module) Class.forName(moduleClass).newInstance());
+ extraModules.add((Module) Class.forName(moduleClass).newInstance());
+ }
+ Module combinedModule = Modules.combine(commModule, coreModule);
+ if (extraModules.size() > 0) {
+ OverriddenModuleBuilder overridenModuleBuilder = Modules.override(combinedModule);
+ combinedModule = overridenModuleBuilder.with(extraModules);
}
- List<com.google.inject.Module> modules = new ArrayList<com.google.inject.Module>();
- modules.add((com.google.inject.Module) commModule);
- modules.add((com.google.inject.Module) coreModule);
- modules.addAll(extraModules);
-
- logger.info(
- "Initializing S4 node with : \n- comm module class [{}]\n- comm configuration file [{}]\n- core module class [{}]\n- core configuration file[{}]\n-extra modules: "
- + Arrays.toString(mainArgs.extraModulesClasses.toArray(new String[] {})), new String[] {
- mainArgs.commModuleClass, commConfigString, mainArgs.coreModuleClass, coreConfigString });
+ if (mainArgs.zkConnectionString != null) {
+ mainArgs.extraNamedParameters.add("cluster.zk_address=" + mainArgs.zkConnectionString);
+ }
if (!mainArgs.extraNamedParameters.isEmpty()) {
logger.debug("Adding named parameters for injection : {}",
Arrays.toString(mainArgs.extraNamedParameters.toArray(new String[] {})));
Map<String, String> namedParameters = new HashMap<String, String>();
+
for (String namedParam : mainArgs.extraNamedParameters) {
- namedParameters.put(namedParam.split("[:]")[0].trim(),
- namedParam.substring(namedParam.indexOf(':') + 1).trim());
+ namedParameters.put(namedParam.split("[=]")[0].trim(),
+ namedParam.substring(namedParam.indexOf('=') + 1).trim());
}
- modules.add(new ParametersInjectionModule(namedParameters));
+ combinedModule = Modules.override(combinedModule).with(new ParametersInjectionModule(namedParameters));
}
- injector = Guice.createInjector(modules);
+ injector = Guice.createInjector(combinedModule);
if (mainArgs.appClass != null) {
logger.info("Starting S4 node with single application from class [{}]", mainArgs.appClass);
@@ -151,12 +163,26 @@ public class Main {
@Parameter(names = "-extraModulesClasses", description = "additional configuration modules (they will be instantiated through their constructor without arguments).", variableArity = true, required = false, hidden = true)
List<String> extraModulesClasses = new ArrayList<String>();
- @Parameter(names = "-namedStringParameters", description = "Guice @Named parameters, used when starting in non dynamic mode, for instance for the adapter. Syntax: '-namedStringParameters=name1:value1,name2:value2 etc...'", hidden = true)
+ @Parameter(names = { "-namedStringParameters", "-p" }, description = "Inline configuration parameters, taking precedence over homonymous configuration parameters from configuration files. Syntax: '-namedStringParameters={name1=value1},{name2=value2} '", hidden = false, converter = InlineConfigParameterConverter.class)
List<String> extraNamedParameters = new ArrayList<String>();
@Parameter(names = "-zk", description = "Zookeeper connection string", required = false)
- String zkConnectionString = "localhost:2181";
+ String zkConnectionString;
+
+ }
+ public static class InlineConfigParameterConverter implements IStringConverter<String> {
+
+ @Override
+ public String convert(String arg) {
+ Pattern parameterPattern = Pattern.compile("\\{(\\S+=\\S+)\\}");
+ logger.info("processing inline configuration parameter {}", arg);
+ Matcher parameterMatcher = parameterPattern.matcher(arg);
+ if (!parameterMatcher.find()) {
+ throw new IllegalArgumentException("Cannot understand parameter " + arg);
+ }
+ return parameterMatcher.group(1);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/22961068/subprojects/s4-core/src/main/java/org/apache/s4/core/Module.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/Module.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/Module.java
deleted file mode 100644
index aa84b69..0000000
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/Module.java
+++ /dev/null
@@ -1,65 +0,0 @@
-package org.apache.s4.core;
-
-import java.io.File;
-import java.io.InputStream;
-
-import javax.inject.Provider;
-
-import org.apache.commons.configuration.ConfigurationConverter;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.ConfigurationUtils;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.s4.deploy.DeploymentManager;
-import org.apache.s4.deploy.DistributedDeploymentManager;
-import org.apache.s4.deploy.NoOpDeploymentManager;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Binder;
-import com.google.inject.TypeLiteral;
-import com.google.inject.name.Names;
-import com.google.inject.util.Providers;
-
-/**
- * First level of S4 configuration,
- *
- * @author Leo Neumeyer
- */
-public class Module extends AbstractModule {
-
- protected PropertiesConfiguration config = null;
-
- private void loadProperties(Binder binder) {
-
- try {
- InputStream is = this.getClass().getResourceAsStream("/s4-core.properties");
- config = new PropertiesConfiguration();
- config.load(is);
-
- System.out.println(ConfigurationUtils.toString(config));
- // TODO - validate properties.
-
- /* Make all properties injectable. Do we need this? */
- Names.bindProperties(binder, ConfigurationConverter.getProperties(config));
- } catch (ConfigurationException e) {
- binder.addError(e);
- e.printStackTrace();
- }
- }
-
- @Override
- protected void configure() {
- if (config == null)
- loadProperties(binder());
-
- bind(Server.class).asEagerSingleton();
- bind(DeploymentManager.class).to(NoOpDeploymentManager.class);
- /*
- * Apps dir is searched as follows: The s4.apps.path property in the properties file. The user's current working
- * directory under the subdirectory /bin/apps.
- */
- String appsDir = config.getString("s4.apps.path", System.getProperty("user.dir") + File.separator + "bin"
- + File.separator + "apps");
- bind(String.class).annotatedWith(Names.named("appsDir")).toInstance(appsDir);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/22961068/subprojects/s4-tools/src/main/resources/templates/newApp.README
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/resources/templates/newApp.README b/subprojects/s4-tools/src/main/resources/templates/newApp.README
index a340c19..a174e31 100644
--- a/subprojects/s4-tools/src/main/resources/templates/newApp.README
+++ b/subprojects/s4-tools/src/main/resources/templates/newApp.README
@@ -16,12 +16,12 @@
To build the project:
- "./gradlew" (from the root of the project, this calls the gradle script from the s4 installation)
- A s4 script has been created at the root of the project's directory. It calls the s4 script from your S4 installation.
+ An "s4" script has been created at the root of the project's directory. It calls the s4 script from your S4 installation.
To execute the application in a new S4 cluster:
1. start a ZooKeeper instance "./s4 zkServer"
- 2. define a logical cluster for your application "./s4 newCluster -cluster=<nameOfTheCluster> -nbTasks=<number of partitions> -firstListeningPort=<a port number for the first node, other nodes use an increment on this initial port>"
+ 2. define a logical cluster for your application "./s4 newCluster -cluster=<nameOfTheCluster> -nbTasks=<number of partitions> -flp=<a port number for the first node, other nodes use an increment on this initial port>"
3. start a node and attach it to the cluster "./s4 node -cluster=<nameOfTheCluster>"
4. deploy the application "./s4 deploy -cluster=<nameOfTheCluster>"
@@ -31,7 +31,5 @@
If you want to use a simple adapter process, listening to an external source, converting incoming data into S4 events, and sending that to S4 apps, you can define
your own app that extends the AdapterApp class. Then to start it (for instance):
- - ./s4 adapter -cluster=c1 -appClass=org.apache.s4.example.twitter.TwitterInputAdapter -namedStringParameters=adapter.output.stream:stream1
-
-
+ - ./s4 adapter -cluster=c1 -appClass=org.apache.s4.example.twitter.TwitterInputAdapter -p={adapter.output.stream=stream1}