You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by jg...@apache.org on 2015/04/22 22:18:46 UTC
samza git commit: SAMZA-655: Add environment variable configuration
rewriter.
Repository: samza
Updated Branches:
refs/heads/master 5a035b8c4 -> 88a844b06
SAMZA-655: Add environment variable configuration rewriter.
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/88a844b0
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/88a844b0
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/88a844b0
Branch: refs/heads/master
Commit: 88a844b066c3d438c9e243997ebdd6e4f98b5f5f
Parents: 5a035b8
Author: Dan Harvey <da...@state.com>
Authored: Wed Apr 22 13:18:07 2015 -0700
Committer: Jakob Homan <jg...@gmail.com>
Committed: Wed Apr 22 13:18:07 2015 -0700
----------------------------------------------------------------------
.../versioned/jobs/configuration-table.html | 6 +-
.../samza/config/EnvironmentConfigRewriter.java | 98 ++++++++++++++++++++
.../config/EnvironmentConfigRewriterTest.java | 78 ++++++++++++++++
3 files changed, 181 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/88a844b0/docs/learn/documentation/versioned/jobs/configuration-table.html
----------------------------------------------------------------------
diff --git a/docs/learn/documentation/versioned/jobs/configuration-table.html b/docs/learn/documentation/versioned/jobs/configuration-table.html
index 5ebe8a7..728197d 100644
--- a/docs/learn/documentation/versioned/jobs/configuration-table.html
+++ b/docs/learn/documentation/versioned/jobs/configuration-table.html
@@ -167,12 +167,16 @@
the set of input streams dynamically at runtime. The value of this property is a
fully-qualified Java classname which must implement
<a href="../api/javadocs/org/apache/samza/config/ConfigRewriter.html">ConfigRewriter</a>.
- Samza ships with one rewriter by default:
+ Samza ships with these rewriters by default:
<dl>
<dt><code>org.apache.samza.config.RegExTopicGenerator</code></dt>
<dd>When consuming from Kafka, this allows you to consume all Kafka topics that match
some regular expression (rather than having to list each topic explicitly).
This rewriter has <a href="#regex-rewriter">additional configuration</a>.</dd>
+ <dt><code>org.apache.samza.config.EnvironmentConfigRewriter</code></dt>
+ <dd>This rewriter takes environment variables that are prefixed with <i>SAMZA_</i>
+ and adds them to the configuration, overriding previous values where they
+ exist. The keys are lowercased and underscores are converted to dots.</dd>
</dl>
</td>
</tr>
http://git-wip-us.apache.org/repos/asf/samza/blob/88a844b0/samza-core/src/main/java/org/apache/samza/config/EnvironmentConfigRewriter.java
----------------------------------------------------------------------
diff --git a/samza-core/src/main/java/org/apache/samza/config/EnvironmentConfigRewriter.java b/samza-core/src/main/java/org/apache/samza/config/EnvironmentConfigRewriter.java
new file mode 100644
index 0000000..e48f21d
--- /dev/null
+++ b/samza-core/src/main/java/org/apache/samza/config/EnvironmentConfigRewriter.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.config;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A {@link org.apache.samza.config.ConfigRewriter} that takes system environment variables
+ * and adds them to the existing {@link org.apache.samza.config.Config}. It will over-ride any
+ * existing properties that were set before this rewriter is called.
+ *
+ * <p>
+ * The following mappings are applied to any environment variable that starts with the prefix
+ * {@value org.apache.samza.config.EnvironmentConfigRewriter#SAMZA_KEY_PREFIX}.
+ * </p>
+ *
+ * <ul>
+ * <li>The prefixed of {@value org.apache.samza.config.EnvironmentConfigRewriter#SAMZA_KEY_PREFIX} is stripped.</li>
+ * <li>The remaining key is downcased.</li>
+ * <li>All underscores are converted to full stops.</li>
+ * </ul>
+ *
+ * <p>
+ * For example the environment variable <i>SAMZA_FOO_BAR=baz</i> will be set as
+ * the<i>foo.bar=baz</i> samza property.
+ * </p>
+ *
+ */
+public class EnvironmentConfigRewriter implements ConfigRewriter {
+ public static final String SAMZA_KEY_PREFIX = "SAMZA_";
+ public static final Logger LOGGER = LoggerFactory.getLogger(EnvironmentConfigRewriter.class);
+
+ @Override
+ public Config rewrite(String name, Config config) {
+ return rewrite(config, System.getenv());
+ }
+
+ protected Config rewrite(Config config, Map<String, String> env) {
+ Map<String, String> envConfig = new HashMap<>();
+
+ for (Map.Entry<String, String> entry: env.entrySet()) {
+ if (entry.getKey().startsWith(SAMZA_KEY_PREFIX)) {
+ String key = renameKey(entry.getKey());
+ String value = entry.getValue();
+
+ if (config.containsKey(key)) {
+ LOGGER.info(String.format("Overriding property from environment: %s=%s", key, value));
+ } else {
+ LOGGER.info(String.format("Setting property from environment: %s=%s", key, value));
+ }
+
+ envConfig.put(key, value);
+ }
+ }
+
+ for (Map.Entry<String, String> entry: config.entrySet()) {
+ String key = entry.getKey();
+
+ if (!envConfig.containsKey(key) && envConfig.containsKey(key.toLowerCase())) {
+ throw new IllegalArgumentException(String.format(
+ "Can't override property from environment with mixed case: %s", key));
+ }
+ }
+
+ return new MapConfig(Arrays.asList(config, envConfig));
+ }
+
+ protected static String renameKey(String envName) {
+ if (envName.length() <= SAMZA_KEY_PREFIX.length()) {
+ throw new IllegalArgumentException();
+ }
+
+ String configName = envName.substring(SAMZA_KEY_PREFIX.length());
+ return configName.toLowerCase().replace('_', '.');
+ }
+}
http://git-wip-us.apache.org/repos/asf/samza/blob/88a844b0/samza-core/src/test/java/org/apache/samza/config/EnvironmentConfigRewriterTest.java
----------------------------------------------------------------------
diff --git a/samza-core/src/test/java/org/apache/samza/config/EnvironmentConfigRewriterTest.java b/samza-core/src/test/java/org/apache/samza/config/EnvironmentConfigRewriterTest.java
new file mode 100644
index 0000000..ff78368
--- /dev/null
+++ b/samza-core/src/test/java/org/apache/samza/config/EnvironmentConfigRewriterTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.config;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class EnvironmentConfigRewriterTest {
+ private EnvironmentConfigRewriter rewriter;
+
+ @Before
+ public void setup() {
+ rewriter = new EnvironmentConfigRewriter();
+ }
+
+ @Test
+ public void testRewriteKeyValidKeys() {
+ assertEquals("foo", EnvironmentConfigRewriter.renameKey("SAMZA_FOO"));
+ assertEquals("foo.bar", EnvironmentConfigRewriter.renameKey("SAMZA_FOO_BAR"));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRewriteKeyInvalidKeyPrefix() {
+ EnvironmentConfigRewriter.renameKey("SAMZA");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRewriteKeyInvalidKeyNoSubkey() {
+ EnvironmentConfigRewriter.renameKey("SAMZA");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRewriteFailsOnDowncaseMatch() throws Exception {
+ Map<String, String> config = createMap("foo.Bar", "a");
+ Map<String, String> env = createMap("SAMZA_FOO_BAR", "b");
+
+ rewriter.rewrite(new MapConfig(config), env);
+ }
+
+ @Test
+ public void testRewriteOverridesConfig() throws Exception {
+ Map<String, String> config = createMap("foo.bar", "a");
+ Map<String, String> env = createMap("SAMZA_FOO_BAR", "b");
+
+ Config rewritten = rewriter.rewrite(new MapConfig(config), env);
+
+ assertEquals("b", rewritten.get("foo.bar"));
+ }
+
+ private static Map<String, String> createMap(String key, String value) {
+ Map<String, String> map = new HashMap<>();
+ map.put(key, value);
+
+ return map;
+ }
+}