You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by be...@apache.org on 2017/01/06 18:45:39 UTC
flume git commit: FLUME-2993. Add support for environment variables
in configuration files
Repository: flume
Updated Branches:
refs/heads/trunk 2ff2dbbd1 -> 5e52ac4ad
FLUME-2993. Add support for environment variables in configuration files
Flume does not currently support environment variable interpolation in the properties file configuration.
Enabling it helps with:
* removing security credentials from config files
* copy-pastes in configuration files when defining multiple agents
This closes #97
Reviewers: Lior Zeno, Jeff Holoman, Shang Wu
(Bessenyei Bal�zs Don�t via Bessenyei Bal�zs Don�t)
Project: http://git-wip-us.apache.org/repos/asf/flume/repo
Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/5e52ac4a
Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/5e52ac4a
Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/5e52ac4a
Branch: refs/heads/trunk
Commit: 5e52ac4ad18a7776f4c74499c1314fbfd3513160
Parents: 2ff2dbb
Author: Bessenyei Bal�zs Don�t <be...@apache.org>
Authored: Fri Jan 6 18:32:50 2017 +0000
Committer: Bessenyei Bal�zs Don�t <be...@apache.org>
Committed: Fri Jan 6 18:37:14 2017 +0000
----------------------------------------------------------------------
flume-ng-doc/sphinx/FlumeUserGuide.rst | 19 ++++++
flume-ng-node/pom.xml | 7 ++
.../flume/node/EnvVarResolverProperties.java | 60 +++++++++++++++++
.../PropertiesFileConfigurationProvider.java | 13 +++-
.../node/TestEnvVarResolverProperties.java | 69 ++++++++++++++++++++
.../flume-conf-with-envvars.properties | 35 ++++++++++
6 files changed, 202 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-doc/sphinx/FlumeUserGuide.rst
----------------------------------------------------------------------
diff --git a/flume-ng-doc/sphinx/FlumeUserGuide.rst b/flume-ng-doc/sphinx/FlumeUserGuide.rst
index 3c316c6..6bc9dba 100644
--- a/flume-ng-doc/sphinx/FlumeUserGuide.rst
+++ b/flume-ng-doc/sphinx/FlumeUserGuide.rst
@@ -234,6 +234,25 @@ The original Flume terminal will output the event in a log message.
Congratulations - you've successfully configured and deployed a Flume agent! Subsequent sections cover agent configuration in much more detail.
+Using environment variables in configuration files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Flume has the ability to substitute environment variables in the configuration. For example::
+
+ a1.sources = r1
+ a1.sources.r1.type = netcat
+ a1.sources.r1.bind = 0.0.0.0
+ a1.sources.r1.port = ${NC_PORT}
+ a1.sources.r1.channels = c1
+
+NB: it currently works for values only, not for keys. (Ie. only on the "right side" of the `=` mark of the config lines.)
+
+This can be enabled via Java system properties on agent invocation by setting `propertiesImplementation = org.apache.flume.node.EnvVarResolverProperties`.
+
+For example::
+ $ NC_PORT=44444 bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger=INFO,console -DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties
+
+Note the above is just an example, environment variables can be configured in other ways, including being set in `conf/flume-env.sh`.
+
Logging raw data
~~~~~~~~~~~~~~~~
http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/pom.xml
----------------------------------------------------------------------
diff --git a/flume-ng-node/pom.xml b/flume-ng-node/pom.xml
index 6d1067d..f4fb958 100644
--- a/flume-ng-node/pom.xml
+++ b/flume-ng-node/pom.xml
@@ -145,6 +145,13 @@
</dependency>
<dependency>
+ <groupId>com.github.stefanbirkner</groupId>
+ <artifactId>system-rules</artifactId>
+ <version>1.16.0</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java
----------------------------------------------------------------------
diff --git a/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java b/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java
new file mode 100644
index 0000000..e0b0d22
--- /dev/null
+++ b/flume-ng-node/src/main/java/org/apache/flume/node/EnvVarResolverProperties.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flume.node;
+
+import com.google.common.base.Preconditions;
+
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+/**
+ * A class that extends the Java built-in Properties overriding
+ * {@link java.util.Properties#getProperty(String)} to allow ${ENV_VAR_NAME}-style environment
+ * variable inclusions
+ */
+public class EnvVarResolverProperties extends Properties {
+ /**
+ * @param input The input string with ${ENV_VAR_NAME}-style environment variable names
+ * @return The output string with ${ENV_VAR_NAME} replaced with their environment variable values
+ */
+ protected static String resolveEnvVars(String input) {
+ Preconditions.checkNotNull(input);
+ // match ${ENV_VAR_NAME}
+ Pattern p = Pattern.compile("\\$\\{(\\w+)\\}");
+ Matcher m = p.matcher(input);
+ StringBuffer sb = new StringBuffer();
+ while (m.find()) {
+ String envVarName = m.group(1);
+ String envVarValue = System.getenv(envVarName);
+ m.appendReplacement(sb, null == envVarValue ? "" : envVarValue);
+ }
+ m.appendTail(sb);
+ return sb.toString();
+ }
+
+ /**
+ * @param key the property key
+ * @return the value of the property key with ${ENV_VAR_NAME}-style environment variables replaced
+ */
+ @Override
+ public String getProperty(String key) {
+ return resolveEnvVars(super.getProperty(key));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java b/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java
index b428c9e..75f45db 100644
--- a/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java
+++ b/flume-ng-node/src/main/java/org/apache/flume/node/PropertiesFileConfigurationProvider.java
@@ -171,6 +171,7 @@ public class PropertiesFileConfigurationProvider extends
private static final Logger LOGGER = LoggerFactory
.getLogger(PropertiesFileConfigurationProvider.class);
+ private static final String DEFAULT_PROPERTIES_IMPLEMENTATION = "java.util.Properties";
private final File file;
@@ -184,12 +185,22 @@ public class PropertiesFileConfigurationProvider extends
BufferedReader reader = null;
try {
reader = new BufferedReader(new FileReader(file));
- Properties properties = new Properties();
+ String resolverClassName = System.getProperty("propertiesImplementation",
+ DEFAULT_PROPERTIES_IMPLEMENTATION);
+ Class<? extends Properties> propsclass = Class.forName(resolverClassName)
+ .asSubclass(Properties.class);
+ Properties properties = propsclass.newInstance();
properties.load(reader);
return new FlumeConfiguration(toMap(properties));
} catch (IOException ex) {
LOGGER.error("Unable to load file:" + file
+ " (I/O failure) - Exception follows.", ex);
+ } catch (ClassNotFoundException e) {
+ LOGGER.error("Configuration resolver class not found", e);
+ } catch (InstantiationException e) {
+ LOGGER.error("Instantiation exception", e);
+ } catch (IllegalAccessException e) {
+ LOGGER.error("Illegal access exception", e);
} finally {
if (reader != null) {
try {
http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java
----------------------------------------------------------------------
diff --git a/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java b/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java
new file mode 100644
index 0000000..27aeef1
--- /dev/null
+++ b/flume-ng-node/src/test/java/org/apache/flume/node/TestEnvVarResolverProperties.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flume.node;
+
+import junit.framework.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.contrib.java.lang.system.EnvironmentVariables;
+
+import java.io.File;
+
+public class TestEnvVarResolverProperties {
+ private static final File TESTFILE = new File(
+ TestEnvVarResolverProperties.class.getClassLoader()
+ .getResource("flume-conf-with-envvars.properties").getFile());
+
+ @Rule
+ public final EnvironmentVariables environmentVariables = new EnvironmentVariables();
+ private PropertiesFileConfigurationProvider provider;
+
+ @Before
+ public void setUp() throws Exception {
+ provider = new PropertiesFileConfigurationProvider("a1", TESTFILE);
+ }
+
+ @Test
+ public void resolveEnvVar() throws Exception {
+ environmentVariables.set("VARNAME", "varvalue");
+ String resolved = EnvVarResolverProperties.resolveEnvVars("padding ${VARNAME} padding");
+ Assert.assertEquals("padding varvalue padding", resolved);
+ }
+
+ @Test
+ public void resolveEnvVars() throws Exception {
+ environmentVariables.set("VARNAME1", "varvalue1");
+ environmentVariables.set("VARNAME2", "varvalue2");
+ String resolved = EnvVarResolverProperties
+ .resolveEnvVars("padding ${VARNAME1} ${VARNAME2} padding");
+ Assert.assertEquals("padding varvalue1 varvalue2 padding", resolved);
+ }
+
+ @Test
+ public void getProperty() throws Exception {
+ String NC_PORT = "6667";
+ environmentVariables.set("NC_PORT", NC_PORT);
+ System.setProperty("propertiesImplementation",
+ "org.apache.flume.node.EnvVarResolverProperties");
+
+ Assert.assertEquals(NC_PORT, provider.getFlumeConfiguration()
+ .getConfigurationFor("a1")
+ .getSourceContext().get("r1").getParameters().get("port"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flume/blob/5e52ac4a/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties
----------------------------------------------------------------------
diff --git a/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties b/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties
new file mode 100644
index 0000000..536eb02
--- /dev/null
+++ b/flume-ng-node/src/test/resources/flume-conf-with-envvars.properties
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+a1.sources = r1
+a1.sources.r1.type = netcat
+a1.sources.r1.bind = 0.0.0.0
+a1.sources.r1.port = ${NC_PORT}
+a1.sources.r1.channels = c1
+
+a1.channels = c1
+a1.channels.c1.type = memory
+a1.channels.c1.capacity = 10000
+a1.channels.c1.transactionCapacity = 10000
+a1.channels.c1.byteCapacityBufferPercentage = 20
+a1.channels.c1.byteCapacity = 800000
+
+a1.channels = c1
+a1.sinks = k1
+a1.sinks.k1.type = logger
+a1.sinks.k1.channel = c1