You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by br...@apache.org on 2012/05/06 18:24:37 UTC
svn commit: r1334679 - in /incubator/flume/trunk:
flume-ng-configuration/src/main/java/org/apache/flume/
flume-ng-configuration/src/main/java/org/apache/flume/conf/
flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/
flume-ng-configuration...
Author: brock
Date: Sun May 6 16:24:36 2012
New Revision: 1334679
URL: http://svn.apache.org/viewvc?rev=1334679&view=rev
Log:
FLUME-1181: Context must enforce dot-separated prefix for sub-properties
(Arvind Prabhakar via Brock Noland)
Added:
incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/BasicConfigurationConstants.java
incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogSourceConfigurationConstants.java
incubator/flume/trunk/flume-ng-sdk/src/test/resources/
incubator/flume/trunk/flume-ng-sdk/src/test/resources/log4j.properties
Modified:
incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/Context.java
incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java
incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/FlumeConfiguration.java
incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkGroupConfiguration.java
incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceConfiguration.java
incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelSelectorFactory.java
incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/MultiplexingChannelSelector.java
incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java
incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java
incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java
incubator/flume/trunk/flume-ng-node/src/main/java/org/apache/flume/conf/properties/PropertiesFileConfigurationProvider.java
incubator/flume/trunk/flume-ng-node/src/test/java/org/apache/flume/conf/properties/TestPropertiesFileConfigurationProvider.java
incubator/flume/trunk/flume-ng-node/src/test/resources/flume-conf.properties
Modified: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/Context.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/Context.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/Context.java (original)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/Context.java Sun May 6 16:24:36 2012
@@ -23,6 +23,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
@@ -71,12 +72,19 @@ public class Context {
* { key = value}
* </code>
*
+ * <b>Note:</b> The <tt>prefix</tt> must end with a period character. If not
+ * this method will raise an IllegalArgumentException.
+ *
* @param prefix key prefix to find and remove from keys in resulting map
* @return map with keys which matched prefix with prefix removed from
* keys in resulting map. If no keys are matched, the returned map is
* empty
+ * @throws IllegalArguemntException if the given prefix does not end with
+ * a period character.
*/
public ImmutableMap<String, String> getSubProperties(String prefix) {
+ Preconditions.checkArgument(prefix.endsWith("."),
+ "The given prefix does not end with a period (" + prefix + ")");
Map<String, String> result = Maps.newHashMap();
synchronized(parameters) {
for (String key : parameters.keySet()) {
Added: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/BasicConfigurationConstants.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/BasicConfigurationConstants.java?rev=1334679&view=auto
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/BasicConfigurationConstants.java (added)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/BasicConfigurationConstants.java Sun May 6 16:24:36 2012
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.flume.conf;
+
+public final class BasicConfigurationConstants {
+
+ public static final String CONFIG_SOURCES = "sources";
+ public static final String CONFIG_SOURCES_PREFIX = CONFIG_SOURCES + ".";
+ public static final String CONFIG_SOURCE_CHANNELSELECTOR_PREFIX = "selector.";
+
+
+ public static final String CONFIG_SINKS = "sinks";
+ public static final String CONFIG_SINKS_PREFIX = CONFIG_SINKS + ".";
+ public static final String CONFIG_SINK_PROCESSOR_PREFIX = "processor.";
+
+ public static final String CONFIG_SINKGROUPS = "sinkgroups";
+ public static final String CONFIG_SINKGROUPS_PREFIX = CONFIG_SINKGROUPS + ".";
+
+ public static final String CONFIG_CHANNEL = "channel";
+ public static final String CONFIG_CHANNELS = "channels";
+ public static final String CONFIG_CHANNELS_PREFIX = CONFIG_CHANNELS + ".";
+
+ public static final String CONFIG_CONFIG = "config";
+ public static final String CONFIG_TYPE = "type";
+
+ private BasicConfigurationConstants() {
+ // disable explicit object creation
+ }
+
+}
Modified: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java (original)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java Sun May 6 16:24:36 2012
@@ -62,14 +62,15 @@ public abstract class ComponentConfigura
public void configure(Context context) throws ConfigurationException {
failIfConfigured();
- String confType = context.getString(FlumeConfiguration.CONF_TYPE);
+ String confType = context.getString(
+ BasicConfigurationConstants.CONFIG_TYPE);
if (confType != null && !confType.isEmpty()){
this.type = confType;
}
// Type can be set by child class constructors, so check if it was.
if (this.type == null || this.type.isEmpty()) {
errors.add(new FlumeConfigurationError(componentName,
- FlumeConfiguration.CONF_TYPE,
+ BasicConfigurationConstants.CONFIG_TYPE,
FlumeConfigurationErrorType.ATTRS_MISSING, ErrorOrWarning.ERROR));
throw new ConfigurationException(
Modified: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/FlumeConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/FlumeConfiguration.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/FlumeConfiguration.java (original)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/FlumeConfiguration.java Sun May 6 16:24:36 2012
@@ -62,21 +62,11 @@ public class FlumeConfiguration {
private static final Logger logger = LoggerFactory
.getLogger(FlumeConfiguration.class);
- public static final String SOURCES = "sources";
- public static final String SOURCES_PREFIX = SOURCES + ".";
- public static final String SINKS = "sinks";
- public static final String SINKS_PREFIX = SINKS + ".";
- public static final String SINKGROUPS = "sinkgroups";
- public static final String SINKGROUPS_PREFIX = SINKGROUPS + ".";
- public static final String CHANNELS = "channels";
- public static final String CHANNELS_PREFIX = CHANNELS + ".";
private final Map<String, AgentConfiguration> agentConfigMap;
private final LinkedList<FlumeConfigurationError> errors;
public static final String NEWLINE = System.getProperty("line.separator",
"\n");
public static final String INDENTSTEP = " ";
- public static final String CONF_TYPE = "type";
- public static final String CONF_CONFIG = "config";
/**
* Creates a populated Flume Configuration object.
@@ -290,7 +280,7 @@ public class FlumeConfiguration {
logger.warn("Agent configuration for '" + agentName
+ "' does not contain any channels. Marking it as invalid.");
errorList.add(new FlumeConfigurationError(agentName,
- CHANNELS,
+ BasicConfigurationConstants.CONFIG_CHANNELS,
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.ERROR));
return false;
@@ -307,7 +297,7 @@ public class FlumeConfiguration {
logger.warn("Agent configuration for '" + agentName
+ "' does not contain any valid channels. Marking it as invalid.");
errorList.add(new FlumeConfigurationError(agentName,
- FlumeConfiguration.CHANNELS,
+ BasicConfigurationConstants.CONFIG_CHANNELS,
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.ERROR));
return false;
@@ -322,11 +312,11 @@ public class FlumeConfiguration {
logger.warn("Agent configuration for '" + agentName
+ "' has no sources or sinks. Will be marked invalid.");
errorList.add(new FlumeConfigurationError(agentName,
- FlumeConfiguration.SOURCES,
+ BasicConfigurationConstants.CONFIG_SOURCES,
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.ERROR));
errorList.add(new FlumeConfigurationError(agentName,
- FlumeConfiguration.SINKS,
+ BasicConfigurationConstants.CONFIG_SINKS,
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.ERROR));
return false;
@@ -414,13 +404,14 @@ public class FlumeConfiguration {
// Context exists in map.
if (channelContext != null) {
// Get the configuration object for the channel:
- ChannelType chType =
- getKnownChannel(channelContext.getString(CONF_TYPE));
+ ChannelType chType = getKnownChannel(channelContext.getString(
+ BasicConfigurationConstants.CONFIG_TYPE));
boolean configSpecified = false;
String config = null;
// Not a known channel - cannot do specific validation to this channel
if (chType == null) {
- config = channelContext.getString(CONF_CONFIG);
+ config = channelContext.getString
+ (BasicConfigurationConstants.CONFIG_CONFIG);
if (config == null || config.isEmpty()) {
config = "OTHER";
} else {
@@ -477,7 +468,7 @@ public class FlumeConfiguration {
logger.warn("Agent configuration for '" + agentName
+ "' has no sources.");
errorList.add(new FlumeConfigurationError(agentName,
- FlumeConfiguration.SOURCES,
+ BasicConfigurationConstants.CONFIG_SOURCES,
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.WARNING));
return new HashSet<String>();
@@ -512,9 +503,11 @@ public class FlumeConfiguration {
String config = null;
boolean configSpecified = false;
if (srcContext != null) {
- SourceType srcType = getKnownSource(srcContext.getString(CONF_TYPE));
+ SourceType srcType = getKnownSource(srcContext.getString(
+ BasicConfigurationConstants.CONFIG_TYPE));
if (srcType == null) {
- config = srcContext.getString(CONF_CONFIG);
+ config = srcContext.getString(
+ BasicConfigurationConstants.CONFIG_CONFIG);
if (config == null || config.isEmpty()) {
config = "OTHER";
} else {
@@ -578,7 +571,7 @@ public class FlumeConfiguration {
logger.warn("Agent configuration for '" + agentName
+ "' has no sinks.");
errorList.add(new FlumeConfigurationError(agentName,
- FlumeConfiguration.SINKS,
+ BasicConfigurationConstants.CONFIG_SINKS,
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.WARNING));
return new HashSet<String>();
@@ -617,9 +610,11 @@ public class FlumeConfiguration {
} else {
String config = null;
boolean configSpecified = false;
- SinkType sinkType = getKnownSink(sinkContext.getString(CONF_TYPE));
+ SinkType sinkType = getKnownSink(sinkContext.getString(
+ BasicConfigurationConstants.CONFIG_TYPE));
if (sinkType == null) {
- config = sinkContext.getString(CONF_CONFIG);
+ config = sinkContext.getString(
+ BasicConfigurationConstants.CONFIG_CONFIG);
if (config == null || config.isEmpty()) {
config = "OTHER";
} else{
@@ -864,7 +859,7 @@ public class FlumeConfiguration {
private boolean addProperty(String key, String value) {
// Check for sources
- if (key.equals(SOURCES)) {
+ if (key.equals(BasicConfigurationConstants.CONFIG_SOURCES)) {
if (sources == null) {
sources = value;
return true;
@@ -872,7 +867,7 @@ public class FlumeConfiguration {
logger
.warn("Duplicate source list specified for agent: " + agentName);
errorList.add(new FlumeConfigurationError(agentName,
- SOURCES,
+ BasicConfigurationConstants.CONFIG_SOURCES,
FlumeConfigurationErrorType.DUPLICATE_PROPERTY,
ErrorOrWarning.ERROR));
return false;
@@ -880,7 +875,7 @@ public class FlumeConfiguration {
}
// Check for sinks
- if (key.equals(SINKS)) {
+ if (key.equals(BasicConfigurationConstants.CONFIG_SINKS)) {
if (sinks == null) {
sinks = value;
logger.info("Added sinks: " + sinks + " Agent: " + this.agentName);
@@ -888,7 +883,7 @@ public class FlumeConfiguration {
} else {
logger.warn("Duplicate sink list specfied for agent: " + agentName);
errorList.add(new FlumeConfigurationError(agentName,
- SINKS,
+ BasicConfigurationConstants.CONFIG_SINKS,
FlumeConfigurationErrorType.DUPLICATE_PROPERTY,
ErrorOrWarning.ERROR));
return false;
@@ -896,7 +891,7 @@ public class FlumeConfiguration {
}
// Check for channels
- if (key.equals(CHANNELS)) {
+ if (key.equals(BasicConfigurationConstants.CONFIG_CHANNELS)) {
if (channels == null) {
channels = value;
@@ -905,7 +900,7 @@ public class FlumeConfiguration {
logger.warn("Duplicate channel list specified for agent: "
+ agentName);
errorList.add(new FlumeConfigurationError(agentName,
- CHANNELS,
+ BasicConfigurationConstants.CONFIG_CHANNELS,
FlumeConfigurationErrorType.DUPLICATE_PROPERTY,
ErrorOrWarning.ERROR));
return false;
@@ -913,7 +908,7 @@ public class FlumeConfiguration {
}
// Check for sinkgroups
- if (key.equals(SINKGROUPS)) {
+ if (key.equals(BasicConfigurationConstants.CONFIG_SINKGROUPS)) {
if (sinkgroups == null) {
sinkgroups = value;
@@ -922,14 +917,15 @@ public class FlumeConfiguration {
logger
.warn("Duplicate sinkgroup list specfied for agent: " + agentName);
errorList.add(new FlumeConfigurationError(agentName,
- SINKGROUPS,
+ BasicConfigurationConstants.CONFIG_SINKGROUPS,
FlumeConfigurationErrorType.DUPLICATE_PROPERTY,
ErrorOrWarning.ERROR));
return false;
}
}
- ComponentNameAndConfigKey cnck = parseConfigKey(key, SOURCES_PREFIX);
+ ComponentNameAndConfigKey cnck = parseConfigKey(key,
+ BasicConfigurationConstants.CONFIG_SOURCES_PREFIX);
if (cnck != null) {
// it is a source
@@ -945,7 +941,8 @@ public class FlumeConfiguration {
return true;
}
- cnck = parseConfigKey(key, CHANNELS_PREFIX);
+ cnck = parseConfigKey(key,
+ BasicConfigurationConstants.CONFIG_CHANNELS_PREFIX);
if (cnck != null) {
// it is a channel
@@ -961,7 +958,8 @@ public class FlumeConfiguration {
return true;
}
- cnck = parseConfigKey(key, SINKS_PREFIX);
+ cnck = parseConfigKey(key,
+ BasicConfigurationConstants.CONFIG_SINKS_PREFIX);
if (cnck != null) {
// it is a sink
@@ -980,7 +978,8 @@ public class FlumeConfiguration {
return true;
}
- cnck = parseConfigKey(key, SINKGROUPS_PREFIX);
+ cnck = parseConfigKey(key,
+ BasicConfigurationConstants.CONFIG_SINKGROUPS_PREFIX);
if (cnck != null) {
String name = cnck.getComponentName();
Modified: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkGroupConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkGroupConfiguration.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkGroupConfiguration.java (original)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkGroupConfiguration.java Sun May 6 16:24:36 2012
@@ -22,13 +22,12 @@ import java.util.List;
import java.util.Map;
import org.apache.flume.Context;
+import org.apache.flume.conf.BasicConfigurationConstants;
import org.apache.flume.conf.ComponentConfiguration;
import org.apache.flume.conf.ComponentConfigurationFactory;
import org.apache.flume.conf.ConfigurationException;
-import org.apache.flume.conf.FlumeConfiguration;
public class SinkGroupConfiguration extends ComponentConfiguration {
- private static final String PROCESSOR_PREFIX = "processor.";
private Context processorContext;
private List<String> sinks;
private SinkProcessorConfiguration processorConf;
@@ -49,13 +48,14 @@ public class SinkGroupConfiguration exte
@Override
public void configure(Context context) throws ConfigurationException {
super.configure(context);
- sinks = Arrays.asList(context.getString("sinks").split("\\s+"));
- Map<String, String> params = context.getSubProperties(PROCESSOR_PREFIX);
+ sinks = Arrays.asList(context.getString(
+ BasicConfigurationConstants.CONFIG_SINKS).split("\\s+"));
+ Map<String, String> params = context.getSubProperties(
+ BasicConfigurationConstants.CONFIG_SINK_PROCESSOR_PREFIX);
processorContext = new Context();
processorContext.putAll(params);
- SinkProcessorType spType =
- getKnownSinkProcessor(
- processorContext.getString(FlumeConfiguration.CONF_TYPE));
+ SinkProcessorType spType = getKnownSinkProcessor(processorContext.getString(
+ BasicConfigurationConstants.CONFIG_TYPE));
if (spType != null) {
processorConf =
Modified: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceConfiguration.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceConfiguration.java (original)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceConfiguration.java Sun May 6 16:24:36 2012
@@ -22,6 +22,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.flume.Context;
+import org.apache.flume.conf.BasicConfigurationConstants;
import org.apache.flume.conf.ComponentConfiguration;
import org.apache.flume.conf.ComponentConfigurationFactory;
import org.apache.flume.conf.ConfigurationException;
@@ -36,7 +37,7 @@ import org.apache.flume.conf.channel.Cha
public class SourceConfiguration extends ComponentConfiguration {
protected Set<String> channels;
protected ChannelSelectorConfiguration selectorConf;
- private static final String CHANNELS_CONF = "channels";
+
public SourceConfiguration(String componentName) {
super(componentName);
}
@@ -52,7 +53,8 @@ public class SourceConfiguration extends
public void configure(Context context) throws ConfigurationException {
super.configure(context);
try {
- String channelList = context.getString(CHANNELS_CONF);
+ String channelList = context.getString(
+ BasicConfigurationConstants.CONFIG_CHANNELS);
if (channelList != null) {
this.channels =
new HashSet<String>(Arrays.asList(channelList.split("\\s+")));
@@ -65,11 +67,11 @@ public class SourceConfiguration extends
+ this.getComponentName());
}
}
- Map<String, String> selectorParams =
- context.getSubProperties("selector.");
+ Map<String, String> selectorParams = context.getSubProperties(
+ BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);
String selType;
if (selectorParams != null && !selectorParams.isEmpty()) {
- selType = selectorParams.get(FlumeConfiguration.CONF_TYPE);
+ selType = selectorParams.get(BasicConfigurationConstants.CONFIG_TYPE);
System.out.println("Loading selector: " + selType);
} else {
selType = ChannelSelectorConfigurationType.REPLICATING.toString();
@@ -85,7 +87,8 @@ public class SourceConfiguration extends
selectorContext.putAll(selectorParams);
String config = null;
if (selectorType == null) {
- config = selectorContext.getString(FlumeConfiguration.CONF_CONFIG);
+ config = selectorContext.getString(
+ BasicConfigurationConstants.CONFIG_CONFIG);
if (config == null || config.isEmpty()) {
config = "OTHER";
}
Modified: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelSelectorFactory.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelSelectorFactory.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelSelectorFactory.java (original)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelSelectorFactory.java Sun May 6 16:24:36 2012
@@ -25,6 +25,7 @@ import org.apache.flume.Channel;
import org.apache.flume.ChannelSelector;
import org.apache.flume.Context;
import org.apache.flume.FlumeException;
+import org.apache.flume.conf.BasicConfigurationConstants;
import org.apache.flume.conf.Configurables;
import org.apache.flume.conf.channel.ChannelSelectorConfiguration;
import org.apache.flume.conf.channel.ChannelSelectorType;
@@ -39,7 +40,8 @@ public class ChannelSelectorFactory {
public static ChannelSelector create(List<Channel> channels,
Map<String, String> config) {
- ChannelSelector selector = getSelectorForType(config.get("type"));
+ ChannelSelector selector = getSelectorForType(config.get(
+ BasicConfigurationConstants.CONFIG_TYPE));
selector.setChannels(channels);
@@ -52,7 +54,7 @@ public class ChannelSelectorFactory {
public static ChannelSelector create(List<Channel> channels,
ChannelSelectorConfiguration conf) {
- String type = "replicating";
+ String type = ChannelSelectorType.REPLICATING.toString();
if (conf != null){
type = conf.getType();
}
Modified: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/MultiplexingChannelSelector.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/MultiplexingChannelSelector.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/MultiplexingChannelSelector.java (original)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/MultiplexingChannelSelector.java Sun May 6 16:24:36 2012
@@ -36,8 +36,8 @@ public class MultiplexingChannelSelector
public static final String CONFIG_MULTIPLEX_HEADER_NAME = "header";
public static final String DEFAULT_MULTIPLEX_HEADER =
"flume.selector.header";
- public static final String CONFIG_PREFIX_MAPPING = "mapping";
- public static final String CONFIG_PREFIX_DEFAULT = "default";
+ public static final String CONFIG_PREFIX_MAPPING = "mapping.";
+ public static final String CONFIG_DEFAULT_CHANNEL = "default";
@SuppressWarnings("unused")
private static final Logger LOG = LoggerFactory
.getLogger(MultiplexingChannelSelector.class);
@@ -84,15 +84,14 @@ public class MultiplexingChannelSelector
}
defaultChannels = getChannelListFromNames(
- context.getString(CONFIG_PREFIX_DEFAULT),
- channelNameMap);
+ context.getString(CONFIG_DEFAULT_CHANNEL), channelNameMap);
if(defaultChannels.isEmpty()){
throw new FlumeException("Default channel list empty");
}
- Map<String, String> mapConfig = context.getSubProperties(
- CONFIG_PREFIX_MAPPING + ".");
+ Map<String, String> mapConfig =
+ context.getSubProperties(CONFIG_PREFIX_MAPPING);
channelMapping = new HashMap<String, List<Channel>>();
Added: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogSourceConfigurationConstants.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogSourceConfigurationConstants.java?rev=1334679&view=auto
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogSourceConfigurationConstants.java (added)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogSourceConfigurationConstants.java Sun May 6 16:24:36 2012
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.source;
+
+public final class SyslogSourceConfigurationConstants {
+
+ public static final String CONFIG_PORT = "port";
+
+ public static final String CONFIG_HOST = "host";
+
+ public static final String CONFIG_FORMAT_PREFIX = "format.";
+
+ public static final String CONFIG_REGEX = "regex";
+
+ public static final String CONFIG_SEARCH = "search";
+
+ public static final String CONFIG_REPLACE = "replace";
+
+ public static final String CONFIG_DATEFORMAT = "dateFormat";
+
+
+ private SyslogSourceConfigurationConstants() {
+ // Disable explicit creation of objects.
+ }
+
+}
Modified: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java (original)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java Sun May 6 16:24:36 2012
@@ -139,11 +139,13 @@ implements EventDrivenSource, Configurab
@Override
public void configure(Context context) {
- Configurables.ensureRequiredNonNull(context, "port");
- port = context.getInteger("port");
- host = context.getString("host");
+ Configurables.ensureRequiredNonNull(context,
+ SyslogSourceConfigurationConstants.CONFIG_PORT);
+ port = context.getInteger(SyslogSourceConfigurationConstants.CONFIG_PORT);
+ host = context.getString(SyslogSourceConfigurationConstants.CONFIG_HOST);
eventSize = context.getInteger("eventSize", SyslogUtils.DEFAULT_SIZE);
- formaterProp = context.getSubProperties("format.");
+ formaterProp = context.getSubProperties(
+ SyslogSourceConfigurationConstants.CONFIG_FORMAT_PREFIX);
}
}
Modified: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java (original)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java Sun May 6 16:24:36 2012
@@ -126,10 +126,12 @@ public class SyslogUDPSource extends Abs
@Override
public void configure(Context context) {
- Configurables.ensureRequiredNonNull(context, "port");
- port = context.getInteger("port");
- host = context.getString("host");
- formaterProp = context.getSubProperties("format.");
+ Configurables.ensureRequiredNonNull(
+ context, SyslogSourceConfigurationConstants.CONFIG_PORT);
+ port = context.getInteger(SyslogSourceConfigurationConstants.CONFIG_PORT);
+ host = context.getString(SyslogSourceConfigurationConstants.CONFIG_HOST);
+ formaterProp = context.getSubProperties(
+ SyslogSourceConfigurationConstants.CONFIG_FORMAT_PREFIX);
}
}
Modified: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java (original)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java Sun May 6 16:24:36 2012
@@ -111,19 +111,27 @@ public class SyslogUtils {
// extend the default header formatter
public void addFormats(Map<String, String> formatProp) {
- if (formatProp.isEmpty() || !formatProp.containsKey("regex")) {
+ if (formatProp.isEmpty() || !formatProp.containsKey(
+ SyslogSourceConfigurationConstants.CONFIG_REGEX)) {
return;
}
SyslogFormater fmt1 = new SyslogFormater();
- fmt1.regexPattern = formatProp.get("regex");
- if (formatProp.containsKey("search")) {
- fmt1.searchPattern.add(formatProp.get("search"));
- }
- if (formatProp.containsKey("replace")) {
- fmt1.replacePattern.add(formatProp.get("replace"));
- }
- if (formatProp.containsKey("dateFormat")) {
- fmt1.dateFormat.add(new SimpleDateFormat(formatProp.get("dateFormat")));
+ fmt1.regexPattern = formatProp.get(
+ SyslogSourceConfigurationConstants.CONFIG_REGEX);
+ if (formatProp.containsKey(
+ SyslogSourceConfigurationConstants.CONFIG_SEARCH)) {
+ fmt1.searchPattern.add(formatProp.get(
+ SyslogSourceConfigurationConstants.CONFIG_SEARCH));
+ }
+ if (formatProp.containsKey(
+ SyslogSourceConfigurationConstants.CONFIG_REPLACE)) {
+ fmt1.replacePattern.add(formatProp.get(
+ SyslogSourceConfigurationConstants.CONFIG_REPLACE));
+ }
+ if (formatProp.containsKey(
+ SyslogSourceConfigurationConstants.CONFIG_DATEFORMAT)) {
+ fmt1.dateFormat.add(new SimpleDateFormat(formatProp.get(
+ SyslogSourceConfigurationConstants.CONFIG_DATEFORMAT)));
}
formats.add(0, fmt1);
}
Modified: incubator/flume/trunk/flume-ng-node/src/main/java/org/apache/flume/conf/properties/PropertiesFileConfigurationProvider.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-node/src/main/java/org/apache/flume/conf/properties/PropertiesFileConfigurationProvider.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-node/src/main/java/org/apache/flume/conf/properties/PropertiesFileConfigurationProvider.java (original)
+++ incubator/flume/trunk/flume-ng-node/src/main/java/org/apache/flume/conf/properties/PropertiesFileConfigurationProvider.java Sun May 6 16:24:36 2012
@@ -39,16 +39,17 @@ import org.apache.flume.Source;
import org.apache.flume.SourceRunner;
import org.apache.flume.channel.ChannelProcessor;
import org.apache.flume.channel.ChannelSelectorFactory;
+import org.apache.flume.conf.BasicConfigurationConstants;
+import org.apache.flume.conf.ComponentConfiguration;
import org.apache.flume.conf.Configurables;
import org.apache.flume.conf.FlumeConfiguration;
+import org.apache.flume.conf.FlumeConfiguration.AgentConfiguration;
import org.apache.flume.conf.channel.ChannelSelectorConfiguration;
import org.apache.flume.conf.file.AbstractFileConfigurationProvider;
import org.apache.flume.conf.file.SimpleNodeConfiguration;
import org.apache.flume.conf.sink.SinkConfiguration;
import org.apache.flume.conf.sink.SinkGroupConfiguration;
import org.apache.flume.conf.source.SourceConfiguration;
-import org.apache.flume.conf.FlumeConfiguration.AgentConfiguration;
-import org.apache.flume.conf.ComponentConfiguration;
import org.apache.flume.node.NodeConfiguration;
import org.apache.flume.sink.DefaultSinkProcessor;
import org.apache.flume.sink.SinkGroup;
@@ -256,7 +257,8 @@ public class PropertiesFileConfiguration
for (String ch : agentConf.getChannelContext().keySet()) {
Context context = agentConf.getChannelContext().get(ch);
Channel channel =
- getChannelFactory().create(ch, context.getString("type"));
+ getChannelFactory().create(ch, context.getString(
+ BasicConfigurationConstants.CONFIG_TYPE));
Configurables.configure(channel, context);
conf.getChannels().put(ch, channel);
LOGGER.info("created channel " + ch);
@@ -298,15 +300,17 @@ public class PropertiesFileConfiguration
Context context = sourceContexts.get(src);
Source source =
getSourceFactory().create(src,
- context.getString("type"));
+ context.getString(BasicConfigurationConstants.CONFIG_TYPE));
List<Channel> channels = new ArrayList<Channel>();
Configurables.configure(source, context);
- String[] channelNames = context.getString("channels").split("\\s+");
+ String[] channelNames = context.getString(
+ BasicConfigurationConstants.CONFIG_CHANNELS).split("\\s+");
for (String chName : channelNames) {
channels.add(conf.getChannels().get(chName));
}
- Map<String, String> selectorConfig = context.getSubProperties("selector" + ".");
+ Map<String, String> selectorConfig = context.getSubProperties(
+ BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);
ChannelSelector selector = ChannelSelectorFactory.create(
channels, selectorConfig);
@@ -339,10 +343,12 @@ public class PropertiesFileConfiguration
Map<String, Context> sinkContexts = agentConf.getSinkContext();
for (String sinkName : sinkContexts.keySet()) {
Context context = sinkContexts.get(sinkName);
- Sink sink = getSinkFactory().create(sinkName, context.getString("type"));
+ Sink sink = getSinkFactory().create(sinkName, context.getString(
+ BasicConfigurationConstants.CONFIG_TYPE));
Configurables.configure(sink, context);
- sink.setChannel(conf.getChannels().get(context.getString("channel")));
+ sink.setChannel(conf.getChannels().get(context.getString(
+ BasicConfigurationConstants.CONFIG_CHANNEL)));
sinks.put(sinkName, sink);
}
Modified: incubator/flume/trunk/flume-ng-node/src/test/java/org/apache/flume/conf/properties/TestPropertiesFileConfigurationProvider.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-node/src/test/java/org/apache/flume/conf/properties/TestPropertiesFileConfigurationProvider.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-node/src/test/java/org/apache/flume/conf/properties/TestPropertiesFileConfigurationProvider.java (original)
+++ incubator/flume/trunk/flume-ng-node/src/test/java/org/apache/flume/conf/properties/TestPropertiesFileConfigurationProvider.java Sun May 6 16:24:36 2012
@@ -20,8 +20,11 @@ package org.apache.flume.conf.properties
import java.io.File;
import org.apache.flume.channel.DefaultChannelFactory;
+import org.apache.flume.node.NodeConfiguration;
+import org.apache.flume.node.nodemanager.NodeConfigurationAware;
import org.apache.flume.sink.DefaultSinkFactory;
import org.apache.flume.source.DefaultSourceFactory;
+import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,9 +39,24 @@ public class TestPropertiesFileConfigura
private static final Logger LOGGER = LoggerFactory
.getLogger(TestPropertiesFileConfigurationProvider.class);
+ @Before
+ public void setUp() throws Exception {
+ File tmpDir = new File("target/test");
+ tmpDir.mkdirs();
+
+ File derbyLogFile = new File(tmpDir, "derbytest.log");
+ String derbyLogFilePath = derbyLogFile.getCanonicalPath();
+
+ System.setProperty("derby.stream.error.file", derbyLogFilePath);
+ }
+
@Test
public void testPropertyRead() throws Exception {
- PropertiesFileConfigurationProvider provider = new PropertiesFileConfigurationProvider();
+ PropertiesFileConfigurationProvider provider =
+ new PropertiesFileConfigurationProvider();
+
+ provider.setNodeName("host1");
+ provider.setConfigurationAware(new DummyNodeConfigurationAware());
provider.setChannelFactory(new DefaultChannelFactory());
provider.setSourceFactory(new DefaultSourceFactory());
@@ -48,4 +66,13 @@ public class TestPropertiesFileConfigura
provider.load();
}
+ private static class DummyNodeConfigurationAware implements
+ NodeConfigurationAware {
+
+ @Override
+ public void onNodeConfigurationChanged(NodeConfiguration config) {
+ // no handling necessary
+ }
+ }
+
}
Modified: incubator/flume/trunk/flume-ng-node/src/test/resources/flume-conf.properties
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-node/src/test/resources/flume-conf.properties?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-node/src/test/resources/flume-conf.properties (original)
+++ incubator/flume/trunk/flume-ng-node/src/test/resources/flume-conf.properties Sun May 6 16:24:36 2012
@@ -23,21 +23,25 @@
# host2, host3 etc.
#
-host1.sources = avroSource thriftSource
-host1.channels = jdbcChannel
+host1.sources = avroSource syslogSource
+host1.channels = jdbcChannel memChannel
host1.sinks = hdfsSink
# avroSource configuration
-host1.sources.avroSource.type = org.apache.flume.source.AvroSource
-host1.sources.avroSource.runner.type = avro
-host1.sources.avroSource.runner.port = 11001
+host1.sources.avroSource.type = avro
+host1.sources.avroSource.bind = 127.0.0.1
+host1.sources.avroSource.port = 11001
host1.sources.avroSource.channels = jdbcChannel
-# thriftSource configuration
-host1.sources.thriftSource.type = org.apache.flume.source.ThriftSource
-host1.sources.thriftSource.runner.type = thrift
-host1.sources.thriftSource.runner.port = 12001
-host1.sources.thriftSource.channels = jdbcChannel
+# syslogSource configuration
+host1.sources.syslogSource.type = syslogtcp
+host1.sources.syslogSource.port = 13231
+host1.sources.syslogSource.channels = jdbcChannel memChannel
+host1.sources.syslogSource.selector.type = multiplexing
+host1.sources.syslogSource.selector.header = my.selector.header
+host1.sources.syslogSource.selector.mapping.all = jdbcChannel memChannel
+host1.sources.syslogSource.selector.mapping.persist = jdbcChannel
+host1.sources.syslogSource.selector.default = memChannel
# jdbcChannel configuration
host1.channels.jdbcChannel.type = jdbc
@@ -46,6 +50,10 @@ host1.channels.jdbcChannel.jdbc.connect.
host1.channels.jdbcChannel.jdbc.username = flume
host1.channels.jdbcChannel.jdbc.password = flume
+# memChannel configuration
+host1.channels.memChannel.type = memory
+host1.channels.memChannel.capacity = 10000
+
# hdfsSink configuration
host1.sinks.hdfsSink.type = hdfs
host1.sinks.hdfsSink.namenode = hdfs://localhost/
Added: incubator/flume/trunk/flume-ng-sdk/src/test/resources/log4j.properties
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-sdk/src/test/resources/log4j.properties?rev=1334679&view=auto
==============================================================================
--- incubator/flume/trunk/flume-ng-sdk/src/test/resources/log4j.properties (added)
+++ incubator/flume/trunk/flume-ng-sdk/src/test/resources/log4j.properties Sun May 6 16:24:36 2012
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger = INFO, out
+
+log4j.appender.out = org.apache.log4j.ConsoleAppender
+log4j.appender.out.layout = org.apache.log4j.PatternLayout
+log4j.appender.out.layout.ConversionPattern = %d (%t) [%p - %l] %m%n
+
+log4j.logger.org.apache.flume = DEBUG