You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by ju...@apache.org on 2012/06/19 11:56:56 UTC

svn commit: r1351630 - in /incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase: AsyncHBaseSink.java AsyncHbaseEventSerializer.java HBaseSink.java HBaseSinkConfigurationConstants.java

Author: juhanic
Date: Tue Jun 19 09:56:56 2012
New Revision: 1351630

URL: http://svn.apache.org/viewvc?rev=1351630&view=rev
Log:
FLUME-1280: Make all config properties of Hbase sinks public constants

(Hari Shreedharan via Juhani Connolly)

Added:
    incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/HBaseSinkConfigurationConstants.java
Modified:
    incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/AsyncHBaseSink.java
    incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/AsyncHbaseEventSerializer.java
    incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/HBaseSink.java

Modified: incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/AsyncHBaseSink.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/AsyncHBaseSink.java?rev=1351630&r1=1351629&r2=1351630&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/AsyncHBaseSink.java (original)
+++ incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/AsyncHBaseSink.java Tue Jun 19 09:56:56 2012
@@ -30,7 +30,6 @@ import org.apache.flume.Event;
 import org.apache.flume.EventDeliveryException;
 import org.apache.flume.Transaction;
 import org.apache.flume.conf.Configurable;
-import org.apache.flume.serialization.EventSerializer;
 import org.apache.flume.sink.AbstractSink;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -55,15 +54,17 @@ import com.stumbleupon.async.Callback;
 * encountered in the classpath. This sink supports batch reading of
 * events from the channel, and writing them to Hbase, to minimize the number
 * of flushes on the hbase tables. To use this sink, it has to be configured
-* with certain mandatory parameters:
+* with certain mandatory parameters:<p>
 *
 * <tt>table: </tt> The name of the table in Hbase to write to. <p>
-* <tt>columnFamily: </tt> The column family in Hbase to write to.
+* <tt>columnFamily: </tt> The column family in Hbase to write to.<p>
 * Other optional parameters are:<p>
-* <tt>serializer:</tt> A class implementing {@link AsyncHBaseEventSerializer}.
+* <tt>serializer:</tt> A class implementing
+*  {@link AsyncHbaseEventSerializer}.
 *  An instance of
 * this class will be used to serialize events which are written to hbase.<p>
-* <tt>serializer.*:</tt> Passed in the configure() method to serializer
+* <tt>serializer.*:</tt> Passed in the <code>configure()</code> method to
+* serializer
 * as an object of {@link org.apache.flume.Context}.<p>
 * <tt>batchSize: </tt>This is the batch size used by the client. This is the
 * maximum number of events the sink will commit per transaction. The default
@@ -197,13 +198,15 @@ public class AsyncHBaseSink extends Abst
 
   @Override
   public void configure(Context context) {
-    tableName = context.getString("table");
-    String cf = context.getString("columnFamily");
-    batchSize = context.getLong("batchSize", new Long(100));
+    tableName = context.getString(HBaseSinkConfigurationConstants.CONFIG_TABLE);
+    String cf = context.getString(
+        HBaseSinkConfigurationConstants.CONFIG_COLUMN_FAMILY);
+    batchSize = context.getLong(
+        HBaseSinkConfigurationConstants.CONFIG_BATCHSIZE, new Long(100));
     serializerContext = new Context();
     //If not specified, will use HBase defaults.
     eventSerializerType = context.getString(
-        "serializer");
+        HBaseSinkConfigurationConstants.CONFIG_SERIALIZER);
     Preconditions.checkNotNull(tableName,
         "Table name cannot be empty, please specify in configuration file");
     Preconditions.checkNotNull(cf,
@@ -214,8 +217,8 @@ public class AsyncHBaseSink extends Abst
           "org.apache.flume.sink.hbase.SimpleAsyncHbaseEventSerializer";
       logger.info("No serializer defined, Will use default");
     }
-    serializerContext.putAll(
-        context.getSubProperties(EventSerializer.CTX_PREFIX));
+    serializerContext.putAll(context.getSubProperties(
+            HBaseSinkConfigurationConstants.CONFIG_SERIALIZER_PREFIX));
     columnFamily = cf.getBytes(Charsets.UTF_8);
     try {
       @SuppressWarnings("unchecked")

Modified: incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/AsyncHbaseEventSerializer.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/AsyncHbaseEventSerializer.java?rev=1351630&r1=1351629&r2=1351630&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/AsyncHbaseEventSerializer.java (original)
+++ incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/AsyncHbaseEventSerializer.java Tue Jun 19 09:56:56 2012
@@ -37,10 +37,16 @@ import org.hbase.async.PutRequest;
 public interface AsyncHbaseEventSerializer extends Configurable,
 ConfigurableComponent {
 
+  /**
+   * Initialize the event serializer.
+   * @param table - The table the serializer should use when creating
+   * {@link org.hbase.async.PutRequest} or
+   * {@link org.hbase.async.AtomicIncrementRequest}.
+   * @param cf - The column family to be used.
+   */
   public void initialize(byte[] table, byte[] cf);
 
   /**
-   * Initialize the event serializer.
    * @param Event to be written to HBase.
    */
   public void setEvent(Event event);
@@ -55,9 +61,17 @@ ConfigurableComponent {
    */
   public List<PutRequest> getActions();
 
+  /**
+   * Get the increments that should be made in hbase as a result of this
+   * event. This list is written to hbase.
+   * @return List of {@link org.hbase.async.AtomicIncrementRequest} which
+   * are written as such to HBase.
+   *
+   *
+   */
   public List<AtomicIncrementRequest> getIncrements();
 
-  /*
+  /**
    * Clean up any state. This will be called when the sink is being stopped.
    */
   public void cleanUp();

Modified: incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/HBaseSink.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/HBaseSink.java?rev=1351630&r1=1351629&r2=1351630&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/HBaseSink.java (original)
+++ incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/HBaseSink.java Tue Jun 19 09:56:56 2012
@@ -30,7 +30,6 @@ import org.apache.flume.EventDeliveryExc
 import org.apache.flume.FlumeException;
 import org.apache.flume.Transaction;
 import org.apache.flume.conf.Configurable;
-import org.apache.flume.serialization.EventSerializer;
 import org.apache.flume.sink.AbstractSink;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -44,6 +43,7 @@ import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
 
+
 /**
  *
  * A simple sink which reads events from a channel and writes them to HBase.
@@ -51,17 +51,12 @@ import com.google.common.base.Throwables
  * encountered in the classpath. This sink supports batch reading of
  * events from the channel, and writing them to Hbase, to minimize the number
  * of flushes on the hbase tables. To use this sink, it has to be configured
- * with certain mandatory parameters:
- *
- * This sink also allows these other parameters: <p>
- * <tt>batchsize: </tt>This is the batch size used by the client. This is the
- * maximum number of events the sink will commit per transaction. The default
- * batch size is 100 events.
- * <p>
- *
+ * with certain mandatory parameters:<p>
+ * <tt>table: </tt> The name of the table in Hbase to write to. <p>
+ * <tt>columnFamily: </tt> The column family in Hbase to write to.<p>
  * This sink will commit each transaction if the table's write buffer size is
  * reached or if the number of events in the current transaction reaches the
- * batch size, whichever comes first.
+ * batch size, whichever comes first.<p>
  * Other optional parameters are:<p>
  * <tt>serializer:</tt> A class implementing {@link HBaseEventSerializer}.
  *  An instance of
@@ -148,13 +143,15 @@ public class HBaseSink extends AbstractS
   @SuppressWarnings("unchecked")
   @Override
   public void configure(Context context){
-    tableName = context.getString("table");
-    String cf = context.getString("columnFamily");
-    batchSize = context.getLong("batchSize", new Long(100));
+    tableName = context.getString(HBaseSinkConfigurationConstants.CONFIG_TABLE);
+    String cf = context.getString(
+        HBaseSinkConfigurationConstants.CONFIG_COLUMN_FAMILY);
+    batchSize = context.getLong(
+        HBaseSinkConfigurationConstants.CONFIG_BATCHSIZE, new Long(100));
     serializerContext = new Context();
     //If not specified, will use HBase defaults.
     eventSerializerType = context.getString(
-        "serializer");
+        HBaseSinkConfigurationConstants.CONFIG_SERIALIZER);
     Preconditions.checkNotNull(tableName,
         "Table name cannot be empty, please specify in configuration file");
     Preconditions.checkNotNull(cf,
@@ -165,8 +162,8 @@ public class HBaseSink extends AbstractS
           "org.apache.flume.sink.hbase.SimpleHbaseEventSerializer";
       logger.info("No serializer defined, Will use default");
     }
-    serializerContext.putAll(
-        context.getSubProperties(EventSerializer.CTX_PREFIX));
+    serializerContext.putAll(context.getSubProperties(
+            HBaseSinkConfigurationConstants.CONFIG_SERIALIZER_PREFIX));
     columnFamily = cf.getBytes(Charsets.UTF_8);
     try {
       Class<? extends HbaseEventSerializer> clazz =

Added: incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/HBaseSinkConfigurationConstants.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/HBaseSinkConfigurationConstants.java?rev=1351630&view=auto
==============================================================================
--- incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/HBaseSinkConfigurationConstants.java (added)
+++ incubator/flume/trunk/flume-ng-sinks/flume-ng-hbase-sink/src/main/java/org/apache/flume/sink/hbase/HBaseSinkConfigurationConstants.java Tue Jun 19 09:56:56 2012
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.sink.hbase;
+/**
+ * Constants used for configuration of HBaseSink and AsyncHBaseSink
+ *
+ */
+public class HBaseSinkConfigurationConstants {
+  /**
+   * The Hbase table which the sink should write to.
+   */
+  public static final String CONFIG_TABLE = "table";
+  /**
+   * The column family which the sink should use.
+   */
+  public static final String CONFIG_COLUMN_FAMILY = "columnFamily";
+  /**
+   * Maximum number of events the sink should take from the channel per
+   * transaction, if available.
+   */
+  public static final String CONFIG_BATCHSIZE = "batchSize";
+  /**
+   * The fully qualified class name of the serializer the sink should use.
+   */
+  public static final String CONFIG_SERIALIZER = "serializer";
+  /**
+   * Configuration to pass to the serializer.
+   */
+  public static final String CONFIG_SERIALIZER_PREFIX = CONFIG_SERIALIZER + ".";
+
+
+}