You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by jd...@apache.org on 2016/04/03 04:13:24 UTC
[1/2] incubator-kudu git commit: Create jar-with-dependency from Kudu
Flume sink
Repository: incubator-kudu
Updated Branches:
refs/heads/branch-0.8.x fb9aed67f -> 6d034756a
Create jar-with-dependency from Kudu Flume sink
This will allow the sink to easily be used with the Flume plugins.d
mechanism.
Change-Id: Ie15f0070f580936e27b32b21e001f7e16d84b136
Reviewed-on: http://gerrit.cloudera.org:8080/2712
Tested-by: Kudu Jenkins
Reviewed-by: Jean-Daniel Cryans
(cherry picked from commit 90af0ea581cb8a2fd4af1f3dcf4c29585f50f719)
Reviewed-on: http://gerrit.cloudera.org:8080/2715
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/bce79d56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/bce79d56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/bce79d56
Branch: refs/heads/branch-0.8.x
Commit: bce79d561c9dff0ad3429f59beb6648e5be811ee
Parents: fb9aed6
Author: Mike Percy <mp...@apache.org>
Authored: Fri Apr 1 19:10:50 2016 -0700
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Sun Apr 3 02:12:11 2016 +0000
----------------------------------------------------------------------
java/kudu-flume-sink/pom.xml | 38 ++++++++++++++++++++++++++++++++------
1 file changed, 32 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/bce79d56/java/kudu-flume-sink/pom.xml
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/pom.xml b/java/kudu-flume-sink/pom.xml
index e7d796b..be93339 100644
--- a/java/kudu-flume-sink/pom.xml
+++ b/java/kudu-flume-sink/pom.xml
@@ -24,29 +24,55 @@
<flume.version>1.6.0</flume.version>
</properties>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven-assembly-plugin.version}</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
<dependencies>
<dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-sdk</artifactId>
- <version>${flume.version}</version>
+ <groupId>org.kududb</groupId>
+ <artifactId>kudu-client</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>${flume.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-configuration</artifactId>
<version>${flume.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.kududb</groupId>
- <artifactId>kudu-client</artifactId>
- <version>${project.version}</version>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
[2/2] incubator-kudu git commit: Add javadoc for Kudu Flume sink
Posted by jd...@apache.org.
Add javadoc for Kudu Flume sink
Change-Id: I5cf5d540f1883528f46806089f0abf5d8ad20faa
Reviewed-on: http://gerrit.cloudera.org:8080/2713
Tested-by: Kudu Jenkins
Reviewed-by: Jean-Daniel Cryans
(cherry picked from commit a6dfbf6de09889d56c39e2c23357799778df9bbb)
Reviewed-on: http://gerrit.cloudera.org:8080/2716
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/6d034756
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/6d034756
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/6d034756
Branch: refs/heads/branch-0.8.x
Commit: 6d034756a6ff1b746887882ad5d8004c73674851
Parents: bce79d5
Author: Mike Percy <mp...@apache.org>
Authored: Fri Apr 1 17:35:12 2016 -0700
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Sun Apr 3 02:12:42 2016 +0000
----------------------------------------------------------------------
.../java/org/kududb/flume/sink/KuduSink.java | 29 +++++++++++++++++---
.../flume/sink/SimpleKuduEventProducer.java | 13 +++++----
2 files changed, 33 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/6d034756/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSink.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSink.java b/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSink.java
index 917c0b3..a6ed7bd 100644
--- a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSink.java
+++ b/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/KuduSink.java
@@ -45,10 +45,31 @@ import org.slf4j.LoggerFactory;
import java.util.List;
/**
- * A simple sink which reads events from a channel and writes them to Kudu.
- * To use this sink, it has to be configured with certain mandatory parameters:<p>
- * <tt>tableName: </tt> The name of the table in Kudu to write to. <p>
- * <tt>masterAddresses: </tt> Comma-separated list of "host:port" pairs of the masters (port optional). <p>
+ * <p>A Flume sink that reads events from a channel and writes them to a Kudu table.
+ *
+ * <p><strong>Flume Kudu Sink configuration parameters</strong>
+ *
+ * <table cellpadding=3 cellspacing=0 border=1>
+ * <tr><th>Property Name</th><th>Default</th><th>Required?</th><th>Description</th></tr>
+ * <tr><td>channel</td><td></td><td>Yes</td><td>The name of the Flume channel to read from.</td></tr>
+ * <tr><td>type</td><td></td><td>Yes</td><td>Component name. Must be {@code org.kududb.flume.sink.KuduSink}</td></tr>
+ * <tr><td>masterAddresses</td><td></td><td>Yes</td><td>Comma-separated list of "host:port" pairs of the Kudu master servers. The port is optional.</td></tr>
+ * <tr><td>tableName</td><td></td><td>Yes</td><td>The name of the Kudu table to write to.</td></tr>
+ * <tr><td>batchSize</td><td>100</td><td>No</td><td>The maximum number of events the sink will attempt to take from the channel per transaction.</td></tr>
+ * <tr><td>ignoreDuplicateRows</td><td>true</td><td>No</td><td>Whether to ignore errors indicating that we attempted to insert duplicate rows into Kudu.</td></tr>
+ * <tr><td>timeoutMillis</td><td>10000</td><td>No</td><td>Timeout period for Kudu write operations, in milliseconds.</td></tr>
+ * <tr><td>producer</td><td>{@link org.kududb.flume.sink.SimpleKuduEventProducer}</td><td>No</td><td>The fully qualified class name of the {@link KuduEventProducer} the sink should use.</td></tr>
+ * <tr><td>producer.*</td><td></td><td>(Varies by event producer)</td><td>Configuration properties to pass to the event producer implementation.</td></tr>
+ * </table>
+ *
+ * <p><strong>Installation</strong>
+ *
+ * <p>After building the sink, in order to use it with Flume, place the file named
+ * <tt>kudu-flume-sink-<em>VERSION</em>-jar-with-dependencies.jar</tt> in the
+ * Flume <tt>plugins.d</tt> directory under <tt>kudu-flume-sink/lib/</tt>.
+ *
+ * <p>For detailed instructions on using Flume's plugins.d mechanism, please see the plugins.d
+ * section of the <a href="https://flume.apache.org/FlumeUserGuide.html#the-plugins-d-directory">Flume User Guide</a>.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/6d034756/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/SimpleKuduEventProducer.java
----------------------------------------------------------------------
diff --git a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/SimpleKuduEventProducer.java b/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/SimpleKuduEventProducer.java
index 69d5564..b5be054 100644
--- a/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/SimpleKuduEventProducer.java
+++ b/java/kudu-flume-sink/src/main/java/org/kududb/flume/sink/SimpleKuduEventProducer.java
@@ -32,12 +32,15 @@ import java.util.Collections;
import java.util.List;
/**
- * A simple serializer that returns puts from an event, by writing the event
- * body into it. The headers are discarded.
+ * <p>A simple serializer that generates one {@link Insert} per {@link Event} by writing the event
+ * body into a BINARY column. The headers are discarded.
*
- * Optional parameters: <p>
- * <tt>payloadColumn:</tt> Which column to put payload in. If it is null,
- * payload will be assumed.<p>
+ * <p><strong>Simple Kudu Event Producer configuration parameters</strong>
+ *
+ * <table cellpadding=3 cellspacing=0 border=1>
+ * <tr><th>Property Name</th><th>Default</th><th>Required?</th><th>Description</th></tr>
+ * <tr><td>producer.payloadColumn</td><td>payload</td><td>No</td><td>The name of the BINARY column to write the Flume the event body to.</td></tr>
+ * </table>
*/
public class SimpleKuduEventProducer implements KuduEventProducer {
private byte[] payload;