You are viewing a plain text version of this content. The canonical link for it is here.
Posted to s4-commits@incubator.apache.org by mm...@apache.org on 2013/06/01 11:33:35 UTC
[45/50] [abbrv] git commit: S4-130 Update and document parameters
initialization from nodes - applicable to csv metrics logging configuration -
also: license update for some xml files
S4-130 Update and document parameters initialization from nodes
- applicable to csv metrics logging configuration
- also: license update for some xml files
Project: http://git-wip-us.apache.org/repos/asf/incubator-s4/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s4/commit/394622b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s4/tree/394622b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s4/diff/394622b2
Branch: refs/heads/master
Commit: 394622b224ebc8eabf26311466cc8f06f27e503e
Parents: 2a5f92c
Author: Matthieu Morel <mm...@apache.org>
Authored: Fri Mar 15 17:28:59 2013 +0100
Committer: Matthieu Morel <mm...@apache.org>
Committed: Fri Mar 15 17:29:51 2013 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/s4/core/S4Node.java | 21 ++++-
.../java/org/apache/s4/core/util/ParsingUtils.java | 66 +++++++++++++++
subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml | 16 ++++
subprojects/s4-edsl/src/main/diezel/s4/s4.xml | 16 ++++
.../src/main/java/org/apache/s4/tools/Deploy.java | 42 +--------
website/content/doc/0.6.0/metrics.md | 9 ++-
6 files changed, 126 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
index 7aafe90..fb8f6e4 100644
--- a/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/S4Node.java
@@ -19,16 +19,20 @@ package org.apache.s4.core;
import java.io.IOException;
import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import org.apache.s4.core.util.ArchiveFetchException;
import org.apache.s4.core.util.ParametersInjectionModule;
+import org.apache.s4.core.util.ParsingUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
import com.google.common.io.Resources;
import com.google.inject.Guice;
import com.google.inject.Injector;
@@ -68,10 +72,15 @@ public class S4Node {
}
});
+ // inject parameter from the command line, including zk string
+ Map<String, String> inlineParameters = Maps.newHashMap(ParsingUtils
+ .convertListArgsToMap(nodeArgs.extraNamedParameters));
+ inlineParameters.put("s4.cluster.zk_address", nodeArgs.zkConnectionString);
+
Injector injector = Guice.createInjector(Modules.override(
new BaseModule(Resources.getResource("default.s4.base.properties").openStream(), nodeArgs.clusterName))
- .with(new ParametersInjectionModule(ImmutableMap.of("s4.cluster.zk_address",
- nodeArgs.zkConnectionString))));
+ .with(new ParametersInjectionModule(inlineParameters)));
+
S4Bootstrap bootstrap = injector.getInstance(S4Bootstrap.class);
try {
bootstrap.start(injector);
@@ -96,5 +105,11 @@ public class S4Node {
@Parameter(names = "-zk", description = "Zookeeper connection string", required = false)
String zkConnectionString = "localhost:2181";
+ @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of "
+ + "inline configuration parameters, taking precedence over homonymous configuration parameters from "
+ + "configuration files. Syntax: '-p=name1=value1,name2=value2 '. "
+ + "NOTE: application parameters should be injected in the application configuration/deployment step."
+ + "Only parameters relevant to the node should be injected here, e.g. metrics logging configuration", hidden = false, converter = ParsingUtils.InlineConfigParameterConverter.class)
+ List<String> extraNamedParameters = new ArrayList<String>();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java
new file mode 100644
index 0000000..37e18de
--- /dev/null
+++ b/subprojects/s4-core/src/main/java/org/apache/s4/core/util/ParsingUtils.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s4.core.util;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.beust.jcommander.IStringConverter;
+import com.beust.jcommander.internal.Maps;
+
+/**
+ * Utilities for parsing command line parameters
+ */
+public class ParsingUtils {
+
+ public static Map<String, String> convertListArgsToMap(List<String> args) {
+ Map<String, String> result = Maps.newHashMap();
+ for (String arg : args) {
+ String[] split = arg.split("[=]");
+ if (!(split.length == 2)) {
+ throw new RuntimeException("Invalid args: " + Arrays.toString(args.toArray(new String[] {})));
+ }
+ result.put(split[0], split[1]);
+ }
+ return result;
+ }
+
+ public static class InlineConfigParameterConverter implements IStringConverter<String> {
+
+ private static Logger logger = LoggerFactory.getLogger(InlineConfigParameterConverter.class);
+
+ @Override
+ public String convert(String arg) {
+ Pattern parameterPattern = Pattern.compile("(\\S+=\\S+)");
+ logger.info("processing inline configuration parameter {}", arg);
+ Matcher parameterMatcher = parameterPattern.matcher(arg);
+ if (!parameterMatcher.find()) {
+ throw new IllegalArgumentException("Cannot understand parameter " + arg);
+ }
+ return parameterMatcher.group(1);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
----------------------------------------------------------------------
diff --git a/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml b/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
index 796a781..e562800 100644
--- a/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
+++ b/subprojects/s4-edsl/src/main/diezel/s4/s4-impl.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.-->
<diezelImplementation xmlns="http://diezel.ericaro.net/2.0.0/">
<package>org.apache.s4.edsl</package>
<name>Builder</name>
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
----------------------------------------------------------------------
diff --git a/subprojects/s4-edsl/src/main/diezel/s4/s4.xml b/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
index 90764b2..c019141 100644
--- a/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
+++ b/subprojects/s4-edsl/src/main/diezel/s4/s4.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. -->
<diezel xmlns="http://diezel.ericaro.net/2.0.0/">
<package>org.apache.s4.edsl</package>
<name>S4DSL</name>
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
index f78b866..695faf3 100644
--- a/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
+++ b/subprojects/s4-tools/src/main/java/org/apache/s4/tools/Deploy.java
@@ -23,13 +23,11 @@ import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.I0Itec.zkclient.ZkClient;
import org.apache.s4.comm.topology.ZNRecordSerializer;
import org.apache.s4.core.util.AppConfig;
+import org.apache.s4.core.util.ParsingUtils;
import org.apache.s4.deploy.DeploymentUtils;
import org.gradle.tooling.BuildLauncher;
import org.gradle.tooling.GradleConnector;
@@ -37,10 +35,8 @@ import org.gradle.tooling.ProgressListener;
import org.gradle.tooling.ProjectConnection;
import org.slf4j.LoggerFactory;
-import com.beust.jcommander.IStringConverter;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
-import com.beust.jcommander.internal.Maps;
import com.google.common.base.Strings;
/**
@@ -86,8 +82,8 @@ public class Deploy extends S4ArgsBase {
.appURI(s4rURI == null ? null : s4rURI.toString())
.customModulesNames(deployArgs.modulesClassesNames)
.customModulesURIs(deployArgs.modulesURIs).appClassName(deployArgs.appClass)
- .namedParameters(convertListArgsToMap(deployArgs.extraNamedParameters)).build(),
- deployArgs.clusterName, false, deployArgs.zkConnectionString);
+ .namedParameters(ParsingUtils.convertListArgsToMap(deployArgs.extraNamedParameters))
+ .build(), deployArgs.clusterName, false, deployArgs.zkConnectionString);
// Explicitly shutdown the JVM since Gradle leaves non-daemon threads running that delay the termination
if (!deployArgs.testMode) {
System.exit(0);
@@ -98,18 +94,6 @@ public class Deploy extends S4ArgsBase {
}
- private static Map<String, String> convertListArgsToMap(List<String> args) {
- Map<String, String> result = Maps.newHashMap();
- for (String arg : args) {
- String[] split = arg.split("[=]");
- if (!(split.length == 2)) {
- throw new RuntimeException("Invalid args: " + Arrays.toString(args.toArray(new String[] {})));
- }
- result.put(split[0], split[1]);
- }
- return result;
- }
-
@Parameters(commandNames = "s4 deploy", commandDescription = "Package and deploy application to S4 cluster", separators = "=")
static class DeployAppArgs extends S4ArgsBase {
@@ -137,7 +121,7 @@ public class Deploy extends S4ArgsBase {
@Parameter(names = { "-modulesClasses", "-emc", "-mc" }, description = "Fully qualified class names of custom modules")
List<String> modulesClassesNames = new ArrayList<String>();
- @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of inline configuration parameters, taking precedence over homonymous configuration parameters from configuration files. Syntax: '-p=name1=value1,name2=value2 '", hidden = false, converter = InlineConfigParameterConverter.class)
+ @Parameter(names = { "-namedStringParameters", "-p" }, description = "Comma-separated list of inline configuration parameters, taking precedence over homonymous configuration parameters from configuration files. Syntax: '-p=name1=value1,name2=value2 '", hidden = false, converter = ParsingUtils.InlineConfigParameterConverter.class)
List<String> extraNamedParameters = new ArrayList<String>();
@Parameter(names = "-testMode", description = "Special mode for regression testing", hidden = true)
@@ -147,24 +131,6 @@ public class Deploy extends S4ArgsBase {
boolean debug = false;
}
- /**
- * Parameters parsing utility.
- *
- */
- public static class InlineConfigParameterConverter implements IStringConverter<String> {
-
- @Override
- public String convert(String arg) {
- Pattern parameterPattern = Pattern.compile("(\\S+=\\S+)");
- logger.info("processing inline configuration parameter {}", arg);
- Matcher parameterMatcher = parameterPattern.matcher(arg);
- if (!parameterMatcher.find()) {
- throw new IllegalArgumentException("Cannot understand parameter " + arg);
- }
- return parameterMatcher.group(1);
- }
- }
-
static class ExecGradle {
public static void exec(File buildFile, String taskName, String[] params, boolean debug) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/394622b2/website/content/doc/0.6.0/metrics.md
----------------------------------------------------------------------
diff --git a/website/content/doc/0.6.0/metrics.md b/website/content/doc/0.6.0/metrics.md
index 70d08d5..cdc2469 100644
--- a/website/content/doc/0.6.0/metrics.md
+++ b/website/content/doc/0.6.0/metrics.md
@@ -24,14 +24,14 @@ You can also monitor your own PEs. Simply add new probes (`Meter`, `Gauge`, etc.
By default, metrics are exposed by each node through JMX.
-The `s4.metrics.config` parameter enables periodic dumps of aggregated statistics to the **console** or to **files** in csv format. This parameter is specified as an application parameter, and must match the following regular expression:
+The `s4.metrics.config` parameter enables periodic dumps of aggregated statistics to the **console** or to **files** in csv format. This parameter is specified as a node or application parameter [^1], and must match the following regular expression:
(csv:.+|console):(\d+):(DAYS|HOURS|MICROSECONDS|MILLISECONDS|MINUTES|NANOSECONDS|SECONDS)
Examples:
# dump metrics to csv files to /path/to/directory every 10 seconds
- # (recommendation: use a clean directory)
+ # (recommendation: use a different and clean directory for each of the nodes)
csv:/path/to/directory:10:SECONDS
# dump metrics to the console every minute
@@ -39,4 +39,7 @@ Examples:
-Reporting to Ganglia or Graphite is not provided out of the box with S4, but it's quite easy to add. You simply have to add the corresponding dependencies to your project and enable reporting to these systems during the initialization of your application. See the [metrics](http://metrics.codahale.com) documentation for more information.
\ No newline at end of file
+Reporting to Ganglia or Graphite is not provided out of the box with S4, but it's quite easy to add. You simply have to add the corresponding dependencies to your project and enable reporting to these systems during the initialization of your application. See the [metrics](http://metrics.codahale.com) documentation for more information.
+
+
+[^1]: csv-based metrics logging are usually specified as a node configuration, and must point to a clean directory, different for each node. Otherwise the metrics logging system will try to create multiple new files with the same name, and depending on your setup, that may not be possible.
\ No newline at end of file