You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by jg...@apache.org on 2015/04/22 22:18:46 UTC

samza git commit: SAMZA-655: Add environment variable configuration rewriter.

Repository: samza
Updated Branches:
  refs/heads/master 5a035b8c4 -> 88a844b06


SAMZA-655: Add environment variable configuration rewriter.


Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/88a844b0
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/88a844b0
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/88a844b0

Branch: refs/heads/master
Commit: 88a844b066c3d438c9e243997ebdd6e4f98b5f5f
Parents: 5a035b8
Author: Dan Harvey <da...@state.com>
Authored: Wed Apr 22 13:18:07 2015 -0700
Committer: Jakob Homan <jg...@gmail.com>
Committed: Wed Apr 22 13:18:07 2015 -0700

----------------------------------------------------------------------
 .../versioned/jobs/configuration-table.html     |  6 +-
 .../samza/config/EnvironmentConfigRewriter.java | 98 ++++++++++++++++++++
 .../config/EnvironmentConfigRewriterTest.java   | 78 ++++++++++++++++
 3 files changed, 181 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/samza/blob/88a844b0/docs/learn/documentation/versioned/jobs/configuration-table.html
----------------------------------------------------------------------
diff --git a/docs/learn/documentation/versioned/jobs/configuration-table.html b/docs/learn/documentation/versioned/jobs/configuration-table.html
index 5ebe8a7..728197d 100644
--- a/docs/learn/documentation/versioned/jobs/configuration-table.html
+++ b/docs/learn/documentation/versioned/jobs/configuration-table.html
@@ -167,12 +167,16 @@
                         the set of input streams dynamically at runtime. The value of this property is a
                         fully-qualified Java classname which must implement
                         <a href="../api/javadocs/org/apache/samza/config/ConfigRewriter.html">ConfigRewriter</a>.
-                        Samza ships with one rewriter by default:
+                        Samza ships with these rewriters by default:
                         <dl>
                             <dt><code>org.apache.samza.config.RegExTopicGenerator</code></dt>
                             <dd>When consuming from Kafka, this allows you to consume all Kafka topics that match
                                 some regular expression (rather than having to list each topic explicitly).
                                 This rewriter has <a href="#regex-rewriter">additional configuration</a>.</dd>
+                            <dt><code>org.apache.samza.config.EnvironmentConfigRewriter</code></dt>
+                            <dd>This rewriter takes environment variables that are prefixed with <i>SAMZA_</i>
+                                and adds them to the configuration, overriding previous values where they
+                                exist. The keys are lowercased and underscores are converted to dots.</dd>
                         </dl>
                     </td>
                 </tr>

http://git-wip-us.apache.org/repos/asf/samza/blob/88a844b0/samza-core/src/main/java/org/apache/samza/config/EnvironmentConfigRewriter.java
----------------------------------------------------------------------
diff --git a/samza-core/src/main/java/org/apache/samza/config/EnvironmentConfigRewriter.java b/samza-core/src/main/java/org/apache/samza/config/EnvironmentConfigRewriter.java
new file mode 100644
index 0000000..e48f21d
--- /dev/null
+++ b/samza-core/src/main/java/org/apache/samza/config/EnvironmentConfigRewriter.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.config;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A {@link org.apache.samza.config.ConfigRewriter} that takes system environment variables
+ * and adds them to the existing {@link org.apache.samza.config.Config}. It will over-ride any
+ * existing properties that were set before this rewriter is called.
+ *
+ * <p>
+ * The following mappings are applied to any environment variable that starts with the prefix
+ * {@value org.apache.samza.config.EnvironmentConfigRewriter#SAMZA_KEY_PREFIX}.
+ * </p>
+ *
+ * <ul>
+ *  <li>The prefixed of {@value org.apache.samza.config.EnvironmentConfigRewriter#SAMZA_KEY_PREFIX} is stripped.</li>
+ *  <li>The remaining key is downcased.</li>
+ *  <li>All underscores are converted to full stops.</li>
+ * </ul>
+ *
+ *  <p>
+ *  For example the environment variable <i>SAMZA_FOO_BAR=baz</i> will be set as
+ *  the<i>foo.bar=baz</i> samza property.
+ *  </p>
+ *
+ */
+public class EnvironmentConfigRewriter implements ConfigRewriter {
+  public static final String SAMZA_KEY_PREFIX = "SAMZA_";
+  public static final Logger LOGGER = LoggerFactory.getLogger(EnvironmentConfigRewriter.class);
+
+  @Override
+  public Config rewrite(String name, Config config) {
+    return rewrite(config, System.getenv());
+  }
+
+  protected Config rewrite(Config config, Map<String, String> env) {
+    Map<String, String> envConfig = new HashMap<>();
+
+    for (Map.Entry<String, String> entry: env.entrySet()) {
+      if (entry.getKey().startsWith(SAMZA_KEY_PREFIX)) {
+        String key = renameKey(entry.getKey());
+        String value = entry.getValue();
+
+        if (config.containsKey(key)) {
+          LOGGER.info(String.format("Overriding property from environment: %s=%s", key, value));
+        } else {
+          LOGGER.info(String.format("Setting property from environment: %s=%s", key, value));
+        }
+
+        envConfig.put(key, value);
+      }
+    }
+
+    for (Map.Entry<String, String> entry: config.entrySet()) {
+      String key = entry.getKey();
+
+      if (!envConfig.containsKey(key) && envConfig.containsKey(key.toLowerCase())) {
+        throw new IllegalArgumentException(String.format(
+            "Can't override property from environment with mixed case: %s", key));
+      }
+    }
+
+    return new MapConfig(Arrays.asList(config, envConfig));
+  }
+
+  protected static String renameKey(String envName) {
+    if (envName.length() <= SAMZA_KEY_PREFIX.length()) {
+      throw new IllegalArgumentException();
+    }
+
+    String configName = envName.substring(SAMZA_KEY_PREFIX.length());
+    return configName.toLowerCase().replace('_', '.');
+  }
+}

http://git-wip-us.apache.org/repos/asf/samza/blob/88a844b0/samza-core/src/test/java/org/apache/samza/config/EnvironmentConfigRewriterTest.java
----------------------------------------------------------------------
diff --git a/samza-core/src/test/java/org/apache/samza/config/EnvironmentConfigRewriterTest.java b/samza-core/src/test/java/org/apache/samza/config/EnvironmentConfigRewriterTest.java
new file mode 100644
index 0000000..ff78368
--- /dev/null
+++ b/samza-core/src/test/java/org/apache/samza/config/EnvironmentConfigRewriterTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.config;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class EnvironmentConfigRewriterTest {
+  private EnvironmentConfigRewriter rewriter;
+
+  @Before
+  public void setup() {
+    rewriter = new EnvironmentConfigRewriter();
+  }
+
+  @Test
+  public void testRewriteKeyValidKeys() {
+    assertEquals("foo", EnvironmentConfigRewriter.renameKey("SAMZA_FOO"));
+    assertEquals("foo.bar", EnvironmentConfigRewriter.renameKey("SAMZA_FOO_BAR"));
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testRewriteKeyInvalidKeyPrefix() {
+    EnvironmentConfigRewriter.renameKey("SAMZA");
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testRewriteKeyInvalidKeyNoSubkey() {
+    EnvironmentConfigRewriter.renameKey("SAMZA");
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testRewriteFailsOnDowncaseMatch() throws Exception {
+    Map<String, String> config = createMap("foo.Bar", "a");
+    Map<String, String> env = createMap("SAMZA_FOO_BAR", "b");
+
+    rewriter.rewrite(new MapConfig(config), env);
+  }
+
+  @Test
+  public void testRewriteOverridesConfig() throws Exception {
+    Map<String, String> config = createMap("foo.bar", "a");
+    Map<String, String> env = createMap("SAMZA_FOO_BAR", "b");
+
+    Config rewritten = rewriter.rewrite(new MapConfig(config), env);
+
+    assertEquals("b", rewritten.get("foo.bar"));
+  }
+
+  private static Map<String, String> createMap(String key, String value) {
+    Map<String, String> map = new HashMap<>();
+    map.put(key, value);
+
+    return map;
+  }
+}