You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by br...@apache.org on 2012/05/06 18:24:37 UTC

svn commit: r1334679 - in /incubator/flume/trunk: flume-ng-configuration/src/main/java/org/apache/flume/ flume-ng-configuration/src/main/java/org/apache/flume/conf/ flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/ flume-ng-configuration...

Author: brock
Date: Sun May  6 16:24:36 2012
New Revision: 1334679

URL: http://svn.apache.org/viewvc?rev=1334679&view=rev
Log:
FLUME-1181: Context must enforce dot-separated prefix for sub-properties

(Arvind Prabhakar via Brock Noland)

Added:
    incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/BasicConfigurationConstants.java
    incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogSourceConfigurationConstants.java
    incubator/flume/trunk/flume-ng-sdk/src/test/resources/
    incubator/flume/trunk/flume-ng-sdk/src/test/resources/log4j.properties
Modified:
    incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/Context.java
    incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java
    incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/FlumeConfiguration.java
    incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkGroupConfiguration.java
    incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceConfiguration.java
    incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelSelectorFactory.java
    incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/MultiplexingChannelSelector.java
    incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java
    incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java
    incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java
    incubator/flume/trunk/flume-ng-node/src/main/java/org/apache/flume/conf/properties/PropertiesFileConfigurationProvider.java
    incubator/flume/trunk/flume-ng-node/src/test/java/org/apache/flume/conf/properties/TestPropertiesFileConfigurationProvider.java
    incubator/flume/trunk/flume-ng-node/src/test/resources/flume-conf.properties

Modified: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/Context.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/Context.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/Context.java (original)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/Context.java Sun May  6 16:24:36 2012
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 
@@ -71,12 +72,19 @@ public class Context {
    * { key = value}
    * </code>
    *
+   * <b>Note:</b> The <tt>prefix</tt> must end with a period character. If not
+   * this method will raise an IllegalArgumentException.
+   *
    * @param prefix key prefix to find and remove from keys in resulting map
    * @return map with keys which matched prefix with prefix removed from
    *   keys in resulting map. If no keys are matched, the returned map is
    *   empty
+   * @throws IllegalArguemntException if the given prefix does not end with
+   *   a period character.
    */
   public ImmutableMap<String, String> getSubProperties(String prefix) {
+    Preconditions.checkArgument(prefix.endsWith("."),
+        "The given prefix does not end with a period (" + prefix + ")");
     Map<String, String> result = Maps.newHashMap();
     synchronized(parameters) {
       for (String key : parameters.keySet()) {

Added: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/BasicConfigurationConstants.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/BasicConfigurationConstants.java?rev=1334679&view=auto
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/BasicConfigurationConstants.java (added)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/BasicConfigurationConstants.java Sun May  6 16:24:36 2012
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.flume.conf;
+
+public final class BasicConfigurationConstants {
+
+  public static final String CONFIG_SOURCES = "sources";
+  public static final String CONFIG_SOURCES_PREFIX = CONFIG_SOURCES + ".";
+  public static final String CONFIG_SOURCE_CHANNELSELECTOR_PREFIX = "selector.";
+
+
+  public static final String CONFIG_SINKS = "sinks";
+  public static final String CONFIG_SINKS_PREFIX = CONFIG_SINKS + ".";
+  public static final String CONFIG_SINK_PROCESSOR_PREFIX = "processor.";
+
+  public static final String CONFIG_SINKGROUPS = "sinkgroups";
+  public static final String CONFIG_SINKGROUPS_PREFIX = CONFIG_SINKGROUPS + ".";
+
+  public static final String CONFIG_CHANNEL = "channel";
+  public static final String CONFIG_CHANNELS = "channels";
+  public static final String CONFIG_CHANNELS_PREFIX = CONFIG_CHANNELS + ".";
+
+  public static final String CONFIG_CONFIG = "config";
+  public static final String CONFIG_TYPE = "type";
+
+  private BasicConfigurationConstants() {
+    // disable explicit object creation
+  }
+
+}

Modified: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java (original)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/ComponentConfiguration.java Sun May  6 16:24:36 2012
@@ -62,14 +62,15 @@ public abstract class ComponentConfigura
 
   public void configure(Context context) throws ConfigurationException {
     failIfConfigured();
-    String confType = context.getString(FlumeConfiguration.CONF_TYPE);
+    String confType = context.getString(
+        BasicConfigurationConstants.CONFIG_TYPE);
     if (confType != null && !confType.isEmpty()){
       this.type = confType;
     }
     // Type can be set by child class constructors, so check if it was.
     if (this.type == null || this.type.isEmpty()) {
       errors.add(new FlumeConfigurationError(componentName,
-          FlumeConfiguration.CONF_TYPE,
+          BasicConfigurationConstants.CONFIG_TYPE,
           FlumeConfigurationErrorType.ATTRS_MISSING, ErrorOrWarning.ERROR));
 
       throw new ConfigurationException(

Modified: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/FlumeConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/FlumeConfiguration.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/FlumeConfiguration.java (original)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/FlumeConfiguration.java Sun May  6 16:24:36 2012
@@ -62,21 +62,11 @@ public class FlumeConfiguration {
   private static final Logger logger = LoggerFactory
       .getLogger(FlumeConfiguration.class);
 
-  public static final String SOURCES = "sources";
-  public static final String SOURCES_PREFIX = SOURCES + ".";
-  public static final String SINKS = "sinks";
-  public static final String SINKS_PREFIX = SINKS + ".";
-  public static final String SINKGROUPS = "sinkgroups";
-  public static final String SINKGROUPS_PREFIX = SINKGROUPS + ".";
-  public static final String CHANNELS = "channels";
-  public static final String CHANNELS_PREFIX = CHANNELS + ".";
   private final Map<String, AgentConfiguration> agentConfigMap;
   private final LinkedList<FlumeConfigurationError> errors;
   public static final String NEWLINE = System.getProperty("line.separator",
       "\n");
   public static final String INDENTSTEP = "  ";
-  public static final String CONF_TYPE = "type";
-  public static final String CONF_CONFIG = "config";
 
   /**
    * Creates a populated Flume Configuration object.
@@ -290,7 +280,7 @@ public class FlumeConfiguration {
         logger.warn("Agent configuration for '" + agentName
             + "' does not contain any channels. Marking it as invalid.");
         errorList.add(new FlumeConfigurationError(agentName,
-            CHANNELS,
+            BasicConfigurationConstants.CONFIG_CHANNELS,
             FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
             ErrorOrWarning.ERROR));
         return false;
@@ -307,7 +297,7 @@ public class FlumeConfiguration {
         logger.warn("Agent configuration for '" + agentName
             + "' does not contain any valid channels. Marking it as invalid.");
         errorList.add(new FlumeConfigurationError(agentName,
-            FlumeConfiguration.CHANNELS,
+            BasicConfigurationConstants.CONFIG_CHANNELS,
             FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
             ErrorOrWarning.ERROR));
         return false;
@@ -322,11 +312,11 @@ public class FlumeConfiguration {
         logger.warn("Agent configuration for '" + agentName
             + "' has no sources or sinks. Will be marked invalid.");
         errorList.add(new FlumeConfigurationError(agentName,
-            FlumeConfiguration.SOURCES,
+            BasicConfigurationConstants.CONFIG_SOURCES,
             FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
             ErrorOrWarning.ERROR));
         errorList.add(new FlumeConfigurationError(agentName,
-            FlumeConfiguration.SINKS,
+            BasicConfigurationConstants.CONFIG_SINKS,
             FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
             ErrorOrWarning.ERROR));
         return false;
@@ -414,13 +404,14 @@ public class FlumeConfiguration {
         // Context exists in map.
         if (channelContext != null) {
           // Get the configuration object for the channel:
-          ChannelType chType =
-              getKnownChannel(channelContext.getString(CONF_TYPE));
+          ChannelType chType = getKnownChannel(channelContext.getString(
+                  BasicConfigurationConstants.CONFIG_TYPE));
           boolean configSpecified = false;
           String config = null;
           // Not a known channel - cannot do specific validation to this channel
           if (chType == null) {
-            config = channelContext.getString(CONF_CONFIG);
+            config = channelContext.getString
+                (BasicConfigurationConstants.CONFIG_CONFIG);
             if (config == null || config.isEmpty()) {
               config = "OTHER";
             } else {
@@ -477,7 +468,7 @@ public class FlumeConfiguration {
         logger.warn("Agent configuration for '" + agentName
             + "' has no sources.");
         errorList.add(new FlumeConfigurationError(agentName,
-            FlumeConfiguration.SOURCES,
+            BasicConfigurationConstants.CONFIG_SOURCES,
             FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
             ErrorOrWarning.WARNING));
         return new HashSet<String>();
@@ -512,9 +503,11 @@ public class FlumeConfiguration {
         String config = null;
         boolean configSpecified = false;
         if (srcContext != null) {
-          SourceType srcType = getKnownSource(srcContext.getString(CONF_TYPE));
+          SourceType srcType = getKnownSource(srcContext.getString(
+              BasicConfigurationConstants.CONFIG_TYPE));
           if (srcType == null) {
-            config = srcContext.getString(CONF_CONFIG);
+            config = srcContext.getString(
+                BasicConfigurationConstants.CONFIG_CONFIG);
             if (config == null || config.isEmpty()) {
               config = "OTHER";
             } else {
@@ -578,7 +571,7 @@ public class FlumeConfiguration {
         logger.warn("Agent configuration for '" + agentName
             + "' has no sinks.");
         errorList.add(new FlumeConfigurationError(agentName,
-            FlumeConfiguration.SINKS,
+            BasicConfigurationConstants.CONFIG_SINKS,
             FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
             ErrorOrWarning.WARNING));
         return new HashSet<String>();
@@ -617,9 +610,11 @@ public class FlumeConfiguration {
         } else {
           String config = null;
           boolean configSpecified = false;
-          SinkType sinkType = getKnownSink(sinkContext.getString(CONF_TYPE));
+          SinkType sinkType = getKnownSink(sinkContext.getString(
+              BasicConfigurationConstants.CONFIG_TYPE));
           if (sinkType == null) {
-            config = sinkContext.getString(CONF_CONFIG);
+            config = sinkContext.getString(
+                BasicConfigurationConstants.CONFIG_CONFIG);
             if (config == null || config.isEmpty()) {
               config = "OTHER";
             } else{
@@ -864,7 +859,7 @@ public class FlumeConfiguration {
 
     private boolean addProperty(String key, String value) {
       // Check for sources
-      if (key.equals(SOURCES)) {
+      if (key.equals(BasicConfigurationConstants.CONFIG_SOURCES)) {
         if (sources == null) {
           sources = value;
           return true;
@@ -872,7 +867,7 @@ public class FlumeConfiguration {
           logger
           .warn("Duplicate source list specified for agent: " + agentName);
           errorList.add(new FlumeConfigurationError(agentName,
-              SOURCES,
+              BasicConfigurationConstants.CONFIG_SOURCES,
               FlumeConfigurationErrorType.DUPLICATE_PROPERTY,
               ErrorOrWarning.ERROR));
           return false;
@@ -880,7 +875,7 @@ public class FlumeConfiguration {
       }
 
       // Check for sinks
-      if (key.equals(SINKS)) {
+      if (key.equals(BasicConfigurationConstants.CONFIG_SINKS)) {
         if (sinks == null) {
           sinks = value;
           logger.info("Added sinks: " + sinks + " Agent: " + this.agentName);
@@ -888,7 +883,7 @@ public class FlumeConfiguration {
         } else {
           logger.warn("Duplicate sink list specfied for agent: " + agentName);
           errorList.add(new FlumeConfigurationError(agentName,
-              SINKS,
+              BasicConfigurationConstants.CONFIG_SINKS,
               FlumeConfigurationErrorType.DUPLICATE_PROPERTY,
               ErrorOrWarning.ERROR));
           return false;
@@ -896,7 +891,7 @@ public class FlumeConfiguration {
       }
 
       // Check for channels
-      if (key.equals(CHANNELS)) {
+      if (key.equals(BasicConfigurationConstants.CONFIG_CHANNELS)) {
         if (channels == null) {
           channels = value;
 
@@ -905,7 +900,7 @@ public class FlumeConfiguration {
           logger.warn("Duplicate channel list specified for agent: "
               + agentName);
           errorList.add(new FlumeConfigurationError(agentName,
-              CHANNELS,
+              BasicConfigurationConstants.CONFIG_CHANNELS,
               FlumeConfigurationErrorType.DUPLICATE_PROPERTY,
               ErrorOrWarning.ERROR));
           return false;
@@ -913,7 +908,7 @@ public class FlumeConfiguration {
       }
 
       // Check for sinkgroups
-      if (key.equals(SINKGROUPS)) {
+      if (key.equals(BasicConfigurationConstants.CONFIG_SINKGROUPS)) {
         if (sinkgroups == null) {
           sinkgroups = value;
 
@@ -922,14 +917,15 @@ public class FlumeConfiguration {
           logger
           .warn("Duplicate sinkgroup list specfied for agent: " + agentName);
           errorList.add(new FlumeConfigurationError(agentName,
-              SINKGROUPS,
+              BasicConfigurationConstants.CONFIG_SINKGROUPS,
               FlumeConfigurationErrorType.DUPLICATE_PROPERTY,
               ErrorOrWarning.ERROR));
           return false;
         }
       }
 
-      ComponentNameAndConfigKey cnck = parseConfigKey(key, SOURCES_PREFIX);
+      ComponentNameAndConfigKey cnck = parseConfigKey(key,
+          BasicConfigurationConstants.CONFIG_SOURCES_PREFIX);
 
       if (cnck != null) {
         // it is a source
@@ -945,7 +941,8 @@ public class FlumeConfiguration {
         return true;
       }
 
-      cnck = parseConfigKey(key, CHANNELS_PREFIX);
+      cnck = parseConfigKey(key,
+          BasicConfigurationConstants.CONFIG_CHANNELS_PREFIX);
 
       if (cnck != null) {
         // it is a channel
@@ -961,7 +958,8 @@ public class FlumeConfiguration {
         return true;
       }
 
-      cnck = parseConfigKey(key, SINKS_PREFIX);
+      cnck = parseConfigKey(key,
+          BasicConfigurationConstants.CONFIG_SINKS_PREFIX);
 
       if (cnck != null) {
         // it is a sink
@@ -980,7 +978,8 @@ public class FlumeConfiguration {
         return true;
       }
 
-      cnck = parseConfigKey(key, SINKGROUPS_PREFIX);
+      cnck = parseConfigKey(key,
+          BasicConfigurationConstants.CONFIG_SINKGROUPS_PREFIX);
 
       if (cnck != null) {
         String name = cnck.getComponentName();

Modified: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkGroupConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkGroupConfiguration.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkGroupConfiguration.java (original)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/sink/SinkGroupConfiguration.java Sun May  6 16:24:36 2012
@@ -22,13 +22,12 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.flume.Context;
+import org.apache.flume.conf.BasicConfigurationConstants;
 import org.apache.flume.conf.ComponentConfiguration;
 import org.apache.flume.conf.ComponentConfigurationFactory;
 import org.apache.flume.conf.ConfigurationException;
-import org.apache.flume.conf.FlumeConfiguration;
 
 public class SinkGroupConfiguration extends ComponentConfiguration {
-  private static final String PROCESSOR_PREFIX = "processor.";
   private Context processorContext;
   private List<String> sinks;
   private SinkProcessorConfiguration processorConf;
@@ -49,13 +48,14 @@ public class SinkGroupConfiguration exte
   @Override
   public void configure(Context context) throws ConfigurationException {
     super.configure(context);
-    sinks = Arrays.asList(context.getString("sinks").split("\\s+"));
-    Map<String, String> params = context.getSubProperties(PROCESSOR_PREFIX);
+    sinks = Arrays.asList(context.getString(
+        BasicConfigurationConstants.CONFIG_SINKS).split("\\s+"));
+    Map<String, String> params = context.getSubProperties(
+        BasicConfigurationConstants.CONFIG_SINK_PROCESSOR_PREFIX);
     processorContext = new Context();
     processorContext.putAll(params);
-    SinkProcessorType spType =
-        getKnownSinkProcessor(
-            processorContext.getString(FlumeConfiguration.CONF_TYPE));
+    SinkProcessorType spType = getKnownSinkProcessor(processorContext.getString(
+            BasicConfigurationConstants.CONFIG_TYPE));
 
     if (spType != null) {
       processorConf =

Modified: incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceConfiguration.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceConfiguration.java (original)
+++ incubator/flume/trunk/flume-ng-configuration/src/main/java/org/apache/flume/conf/source/SourceConfiguration.java Sun May  6 16:24:36 2012
@@ -22,6 +22,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.flume.Context;
+import org.apache.flume.conf.BasicConfigurationConstants;
 import org.apache.flume.conf.ComponentConfiguration;
 import org.apache.flume.conf.ComponentConfigurationFactory;
 import org.apache.flume.conf.ConfigurationException;
@@ -36,7 +37,7 @@ import org.apache.flume.conf.channel.Cha
 public class SourceConfiguration extends ComponentConfiguration {
   protected Set<String> channels;
   protected ChannelSelectorConfiguration selectorConf;
-  private static final String CHANNELS_CONF = "channels";
+
   public SourceConfiguration(String componentName) {
     super(componentName);
   }
@@ -52,7 +53,8 @@ public class SourceConfiguration extends
   public void configure(Context context) throws ConfigurationException {
     super.configure(context);
     try {
-      String channelList = context.getString(CHANNELS_CONF);
+      String channelList = context.getString(
+          BasicConfigurationConstants.CONFIG_CHANNELS);
       if (channelList != null) {
         this.channels =
             new HashSet<String>(Arrays.asList(channelList.split("\\s+")));
@@ -65,11 +67,11 @@ public class SourceConfiguration extends
               + this.getComponentName());
         }
       }
-      Map<String, String> selectorParams =
-          context.getSubProperties("selector.");
+      Map<String, String> selectorParams = context.getSubProperties(
+              BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);
       String selType;
       if (selectorParams != null && !selectorParams.isEmpty()) {
-        selType = selectorParams.get(FlumeConfiguration.CONF_TYPE);
+        selType = selectorParams.get(BasicConfigurationConstants.CONFIG_TYPE);
         System.out.println("Loading selector: " + selType);
       } else {
         selType = ChannelSelectorConfigurationType.REPLICATING.toString();
@@ -85,7 +87,8 @@ public class SourceConfiguration extends
       selectorContext.putAll(selectorParams);
       String config = null;
       if (selectorType == null) {
-        config = selectorContext.getString(FlumeConfiguration.CONF_CONFIG);
+        config = selectorContext.getString(
+            BasicConfigurationConstants.CONFIG_CONFIG);
         if (config == null || config.isEmpty()) {
           config = "OTHER";
         }

Modified: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelSelectorFactory.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelSelectorFactory.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelSelectorFactory.java (original)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/ChannelSelectorFactory.java Sun May  6 16:24:36 2012
@@ -25,6 +25,7 @@ import org.apache.flume.Channel;
 import org.apache.flume.ChannelSelector;
 import org.apache.flume.Context;
 import org.apache.flume.FlumeException;
+import org.apache.flume.conf.BasicConfigurationConstants;
 import org.apache.flume.conf.Configurables;
 import org.apache.flume.conf.channel.ChannelSelectorConfiguration;
 import org.apache.flume.conf.channel.ChannelSelectorType;
@@ -39,7 +40,8 @@ public class ChannelSelectorFactory {
   public static ChannelSelector create(List<Channel> channels,
       Map<String, String> config) {
 
-    ChannelSelector selector = getSelectorForType(config.get("type"));
+    ChannelSelector selector = getSelectorForType(config.get(
+        BasicConfigurationConstants.CONFIG_TYPE));
 
     selector.setChannels(channels);
 
@@ -52,7 +54,7 @@ public class ChannelSelectorFactory {
 
   public static ChannelSelector create(List<Channel> channels,
       ChannelSelectorConfiguration conf) {
-    String type = "replicating";
+    String type = ChannelSelectorType.REPLICATING.toString();
     if (conf != null){
       type = conf.getType();
     }

Modified: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/MultiplexingChannelSelector.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/MultiplexingChannelSelector.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/MultiplexingChannelSelector.java (original)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/channel/MultiplexingChannelSelector.java Sun May  6 16:24:36 2012
@@ -36,8 +36,8 @@ public class MultiplexingChannelSelector
   public static final String CONFIG_MULTIPLEX_HEADER_NAME = "header";
   public static final String DEFAULT_MULTIPLEX_HEADER =
       "flume.selector.header";
-  public static final String CONFIG_PREFIX_MAPPING = "mapping";
-  public static final String CONFIG_PREFIX_DEFAULT = "default";
+  public static final String CONFIG_PREFIX_MAPPING = "mapping.";
+  public static final String CONFIG_DEFAULT_CHANNEL = "default";
   @SuppressWarnings("unused")
   private static final Logger LOG = LoggerFactory
     .getLogger(MultiplexingChannelSelector.class);
@@ -84,15 +84,14 @@ public class MultiplexingChannelSelector
     }
 
     defaultChannels = getChannelListFromNames(
-        context.getString(CONFIG_PREFIX_DEFAULT),
-        channelNameMap);
+        context.getString(CONFIG_DEFAULT_CHANNEL), channelNameMap);
 
     if(defaultChannels.isEmpty()){
       throw new FlumeException("Default channel list empty");
     }
 
-    Map<String, String> mapConfig = context.getSubProperties(
-        CONFIG_PREFIX_MAPPING + ".");
+    Map<String, String> mapConfig =
+        context.getSubProperties(CONFIG_PREFIX_MAPPING);
 
     channelMapping = new HashMap<String, List<Channel>>();
 

Added: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogSourceConfigurationConstants.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogSourceConfigurationConstants.java?rev=1334679&view=auto
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogSourceConfigurationConstants.java (added)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogSourceConfigurationConstants.java Sun May  6 16:24:36 2012
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.source;
+
+public final class SyslogSourceConfigurationConstants {
+
+  public static final String CONFIG_PORT = "port";
+
+  public static final String CONFIG_HOST = "host";
+
+  public static final String CONFIG_FORMAT_PREFIX = "format.";
+
+  public static final String CONFIG_REGEX = "regex";
+
+  public static final String CONFIG_SEARCH = "search";
+
+  public static final String CONFIG_REPLACE = "replace";
+
+  public static final String CONFIG_DATEFORMAT = "dateFormat";
+
+
+  private SyslogSourceConfigurationConstants() {
+    // Disable explicit creation of objects.
+  }
+
+}

Modified: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java (original)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogTcpSource.java Sun May  6 16:24:36 2012
@@ -139,11 +139,13 @@ implements EventDrivenSource, Configurab
 
   @Override
   public void configure(Context context) {
-    Configurables.ensureRequiredNonNull(context, "port");
-    port = context.getInteger("port");
-    host = context.getString("host");
+    Configurables.ensureRequiredNonNull(context,
+        SyslogSourceConfigurationConstants.CONFIG_PORT);
+    port = context.getInteger(SyslogSourceConfigurationConstants.CONFIG_PORT);
+    host = context.getString(SyslogSourceConfigurationConstants.CONFIG_HOST);
     eventSize = context.getInteger("eventSize", SyslogUtils.DEFAULT_SIZE);
-    formaterProp = context.getSubProperties("format.");
+    formaterProp = context.getSubProperties(
+        SyslogSourceConfigurationConstants.CONFIG_FORMAT_PREFIX);
   }
 
 }

Modified: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java (original)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUDPSource.java Sun May  6 16:24:36 2012
@@ -126,10 +126,12 @@ public class SyslogUDPSource extends Abs
 
   @Override
   public void configure(Context context) {
-    Configurables.ensureRequiredNonNull(context, "port");
-    port = context.getInteger("port");
-    host = context.getString("host");
-    formaterProp = context.getSubProperties("format.");
+    Configurables.ensureRequiredNonNull(
+        context, SyslogSourceConfigurationConstants.CONFIG_PORT);
+    port = context.getInteger(SyslogSourceConfigurationConstants.CONFIG_PORT);
+    host = context.getString(SyslogSourceConfigurationConstants.CONFIG_HOST);
+    formaterProp = context.getSubProperties(
+        SyslogSourceConfigurationConstants.CONFIG_FORMAT_PREFIX);
   }
 
 }

Modified: incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java (original)
+++ incubator/flume/trunk/flume-ng-core/src/main/java/org/apache/flume/source/SyslogUtils.java Sun May  6 16:24:36 2012
@@ -111,19 +111,27 @@ public class SyslogUtils {
 
   // extend the default header formatter
   public void addFormats(Map<String, String> formatProp) {
-    if (formatProp.isEmpty() || !formatProp.containsKey("regex")) {
+    if (formatProp.isEmpty() || !formatProp.containsKey(
+        SyslogSourceConfigurationConstants.CONFIG_REGEX)) {
       return;
     }
     SyslogFormater fmt1 = new SyslogFormater();
-    fmt1.regexPattern = formatProp.get("regex");
-    if (formatProp.containsKey("search")) {
-      fmt1.searchPattern.add(formatProp.get("search"));
-    }
-    if (formatProp.containsKey("replace")) {
-      fmt1.replacePattern.add(formatProp.get("replace"));
-    }
-    if (formatProp.containsKey("dateFormat")) {
-        fmt1.dateFormat.add(new SimpleDateFormat(formatProp.get("dateFormat")));
+    fmt1.regexPattern = formatProp.get(
+        SyslogSourceConfigurationConstants.CONFIG_REGEX);
+    if (formatProp.containsKey(
+        SyslogSourceConfigurationConstants.CONFIG_SEARCH)) {
+      fmt1.searchPattern.add(formatProp.get(
+          SyslogSourceConfigurationConstants.CONFIG_SEARCH));
+    }
+    if (formatProp.containsKey(
+        SyslogSourceConfigurationConstants.CONFIG_REPLACE)) {
+      fmt1.replacePattern.add(formatProp.get(
+          SyslogSourceConfigurationConstants.CONFIG_REPLACE));
+    }
+    if (formatProp.containsKey(
+        SyslogSourceConfigurationConstants.CONFIG_DATEFORMAT)) {
+        fmt1.dateFormat.add(new SimpleDateFormat(formatProp.get(
+            SyslogSourceConfigurationConstants.CONFIG_DATEFORMAT)));
     }
     formats.add(0, fmt1);
   }

Modified: incubator/flume/trunk/flume-ng-node/src/main/java/org/apache/flume/conf/properties/PropertiesFileConfigurationProvider.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-node/src/main/java/org/apache/flume/conf/properties/PropertiesFileConfigurationProvider.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-node/src/main/java/org/apache/flume/conf/properties/PropertiesFileConfigurationProvider.java (original)
+++ incubator/flume/trunk/flume-ng-node/src/main/java/org/apache/flume/conf/properties/PropertiesFileConfigurationProvider.java Sun May  6 16:24:36 2012
@@ -39,16 +39,17 @@ import org.apache.flume.Source;
 import org.apache.flume.SourceRunner;
 import org.apache.flume.channel.ChannelProcessor;
 import org.apache.flume.channel.ChannelSelectorFactory;
+import org.apache.flume.conf.BasicConfigurationConstants;
+import org.apache.flume.conf.ComponentConfiguration;
 import org.apache.flume.conf.Configurables;
 import org.apache.flume.conf.FlumeConfiguration;
+import org.apache.flume.conf.FlumeConfiguration.AgentConfiguration;
 import org.apache.flume.conf.channel.ChannelSelectorConfiguration;
 import org.apache.flume.conf.file.AbstractFileConfigurationProvider;
 import org.apache.flume.conf.file.SimpleNodeConfiguration;
 import org.apache.flume.conf.sink.SinkConfiguration;
 import org.apache.flume.conf.sink.SinkGroupConfiguration;
 import org.apache.flume.conf.source.SourceConfiguration;
-import org.apache.flume.conf.FlumeConfiguration.AgentConfiguration;
-import org.apache.flume.conf.ComponentConfiguration;
 import org.apache.flume.node.NodeConfiguration;
 import org.apache.flume.sink.DefaultSinkProcessor;
 import org.apache.flume.sink.SinkGroup;
@@ -256,7 +257,8 @@ public class PropertiesFileConfiguration
     for (String ch : agentConf.getChannelContext().keySet()) {
       Context context = agentConf.getChannelContext().get(ch);
       Channel channel =
-          getChannelFactory().create(ch, context.getString("type"));
+          getChannelFactory().create(ch, context.getString(
+              BasicConfigurationConstants.CONFIG_TYPE));
       Configurables.configure(channel, context);
       conf.getChannels().put(ch, channel);
       LOGGER.info("created channel " + ch);
@@ -298,15 +300,17 @@ public class PropertiesFileConfiguration
       Context context = sourceContexts.get(src);
       Source source =
           getSourceFactory().create(src,
-              context.getString("type"));
+              context.getString(BasicConfigurationConstants.CONFIG_TYPE));
       List<Channel> channels = new ArrayList<Channel>();
       Configurables.configure(source, context);
-      String[] channelNames = context.getString("channels").split("\\s+");
+      String[] channelNames = context.getString(
+          BasicConfigurationConstants.CONFIG_CHANNELS).split("\\s+");
       for (String chName : channelNames) {
         channels.add(conf.getChannels().get(chName));
       }
 
-      Map<String, String> selectorConfig = context.getSubProperties("selector" + ".");
+      Map<String, String> selectorConfig = context.getSubProperties(
+          BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);
 
       ChannelSelector selector = ChannelSelectorFactory.create(
           channels, selectorConfig);
@@ -339,10 +343,12 @@ public class PropertiesFileConfiguration
     Map<String, Context> sinkContexts = agentConf.getSinkContext();
     for (String sinkName : sinkContexts.keySet()) {
       Context context = sinkContexts.get(sinkName);
-      Sink sink = getSinkFactory().create(sinkName, context.getString("type"));
+      Sink sink = getSinkFactory().create(sinkName, context.getString(
+          BasicConfigurationConstants.CONFIG_TYPE));
       Configurables.configure(sink, context);
 
-      sink.setChannel(conf.getChannels().get(context.getString("channel")));
+      sink.setChannel(conf.getChannels().get(context.getString(
+          BasicConfigurationConstants.CONFIG_CHANNEL)));
       sinks.put(sinkName, sink);
     }
 

Modified: incubator/flume/trunk/flume-ng-node/src/test/java/org/apache/flume/conf/properties/TestPropertiesFileConfigurationProvider.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-node/src/test/java/org/apache/flume/conf/properties/TestPropertiesFileConfigurationProvider.java?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-node/src/test/java/org/apache/flume/conf/properties/TestPropertiesFileConfigurationProvider.java (original)
+++ incubator/flume/trunk/flume-ng-node/src/test/java/org/apache/flume/conf/properties/TestPropertiesFileConfigurationProvider.java Sun May  6 16:24:36 2012
@@ -20,8 +20,11 @@ package org.apache.flume.conf.properties
 import java.io.File;
 
 import org.apache.flume.channel.DefaultChannelFactory;
+import org.apache.flume.node.NodeConfiguration;
+import org.apache.flume.node.nodemanager.NodeConfigurationAware;
 import org.apache.flume.sink.DefaultSinkFactory;
 import org.apache.flume.source.DefaultSourceFactory;
+import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,9 +39,24 @@ public class TestPropertiesFileConfigura
   private static final Logger LOGGER = LoggerFactory
       .getLogger(TestPropertiesFileConfigurationProvider.class);
 
+  @Before
+  public void setUp() throws Exception {
+    File tmpDir = new File("target/test");
+    tmpDir.mkdirs();
+
+    File derbyLogFile = new File(tmpDir, "derbytest.log");
+    String derbyLogFilePath = derbyLogFile.getCanonicalPath();
+
+    System.setProperty("derby.stream.error.file", derbyLogFilePath);
+  }
+
   @Test
   public void testPropertyRead() throws Exception {
-    PropertiesFileConfigurationProvider provider = new PropertiesFileConfigurationProvider();
+    PropertiesFileConfigurationProvider provider =
+        new PropertiesFileConfigurationProvider();
+
+    provider.setNodeName("host1");
+    provider.setConfigurationAware(new DummyNodeConfigurationAware());
 
     provider.setChannelFactory(new DefaultChannelFactory());
     provider.setSourceFactory(new DefaultSourceFactory());
@@ -48,4 +66,13 @@ public class TestPropertiesFileConfigura
     provider.load();
   }
 
+  private static class DummyNodeConfigurationAware implements
+    NodeConfigurationAware {
+
+    @Override
+    public void onNodeConfigurationChanged(NodeConfiguration config) {
+       // no handling necessary
+    }
+  }
+
 }

Modified: incubator/flume/trunk/flume-ng-node/src/test/resources/flume-conf.properties
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-node/src/test/resources/flume-conf.properties?rev=1334679&r1=1334678&r2=1334679&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-node/src/test/resources/flume-conf.properties (original)
+++ incubator/flume/trunk/flume-ng-node/src/test/resources/flume-conf.properties Sun May  6 16:24:36 2012
@@ -23,21 +23,25 @@
 # host2, host3 etc.
 #
 
-host1.sources = avroSource thriftSource
-host1.channels = jdbcChannel
+host1.sources = avroSource syslogSource
+host1.channels = jdbcChannel memChannel
 host1.sinks = hdfsSink
 
 # avroSource configuration
-host1.sources.avroSource.type = org.apache.flume.source.AvroSource
-host1.sources.avroSource.runner.type = avro
-host1.sources.avroSource.runner.port = 11001
+host1.sources.avroSource.type = avro
+host1.sources.avroSource.bind = 127.0.0.1
+host1.sources.avroSource.port = 11001
 host1.sources.avroSource.channels = jdbcChannel
 
-# thriftSource configuration
-host1.sources.thriftSource.type = org.apache.flume.source.ThriftSource
-host1.sources.thriftSource.runner.type = thrift
-host1.sources.thriftSource.runner.port = 12001
-host1.sources.thriftSource.channels = jdbcChannel
+# syslogSource configuration
+host1.sources.syslogSource.type = syslogtcp
+host1.sources.syslogSource.port = 13231
+host1.sources.syslogSource.channels = jdbcChannel memChannel
+host1.sources.syslogSource.selector.type = multiplexing
+host1.sources.syslogSource.selector.header = my.selector.header
+host1.sources.syslogSource.selector.mapping.all = jdbcChannel memChannel
+host1.sources.syslogSource.selector.mapping.persist = jdbcChannel
+host1.sources.syslogSource.selector.default = memChannel
 
 # jdbcChannel configuration
 host1.channels.jdbcChannel.type = jdbc
@@ -46,6 +50,10 @@ host1.channels.jdbcChannel.jdbc.connect.
 host1.channels.jdbcChannel.jdbc.username = flume
 host1.channels.jdbcChannel.jdbc.password = flume
 
+# memChannel configuration
+host1.channels.memChannel.type = memory
+host1.channels.memChannel.capacity = 10000
+
 # hdfsSink configuration
 host1.sinks.hdfsSink.type = hdfs
 host1.sinks.hdfsSink.namenode = hdfs://localhost/

Added: incubator/flume/trunk/flume-ng-sdk/src/test/resources/log4j.properties
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-sdk/src/test/resources/log4j.properties?rev=1334679&view=auto
==============================================================================
--- incubator/flume/trunk/flume-ng-sdk/src/test/resources/log4j.properties (added)
+++ incubator/flume/trunk/flume-ng-sdk/src/test/resources/log4j.properties Sun May  6 16:24:36 2012
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+log4j.rootLogger = INFO, out
+
+log4j.appender.out = org.apache.log4j.ConsoleAppender
+log4j.appender.out.layout = org.apache.log4j.PatternLayout
+log4j.appender.out.layout.ConversionPattern = %d (%t) [%p - %l] %m%n
+
+log4j.logger.org.apache.flume = DEBUG