You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2015/05/28 15:36:27 UTC
[1/7] flink git commit: [FLINK-1874] [streaming] Connectors separated
into individual projects
Repository: flink
Updated Branches:
refs/heads/master 72828b5d2 -> 7f5edeb9a
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaITCase.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaITCase.java
deleted file mode 100644
index 7b7bdcc..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaITCase.java
+++ /dev/null
@@ -1,1023 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-import java.util.UUID;
-
-import kafka.admin.AdminUtils;
-import kafka.api.PartitionMetadata;
-import kafka.consumer.Consumer;
-import kafka.consumer.ConsumerConfig;
-import kafka.consumer.ConsumerIterator;
-import kafka.consumer.KafkaStream;
-import kafka.javaapi.consumer.ConsumerConnector;
-import kafka.message.MessageAndMetadata;
-import kafka.network.SocketServer;
-import org.I0Itec.zkclient.ZkClient;
-import org.apache.curator.test.TestingServer;
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.client.JobExecutionException;
-import org.apache.flink.runtime.net.NetUtils;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.DataStreamSource;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
-import org.apache.flink.streaming.api.functions.sink.SinkFunction;
-import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
-import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.connectors.kafka.api.KafkaSink;
-import org.apache.flink.streaming.connectors.kafka.api.KafkaSource;
-import org.apache.flink.streaming.connectors.kafka.api.persistent.PersistentKafkaSource;
-import org.apache.flink.streaming.connectors.kafka.partitioner.SerializableKafkaPartitioner;
-import org.apache.flink.streaming.connectors.kafka.util.KafkaLocalSystemTime;
-import org.apache.flink.streaming.util.serialization.DeserializationSchema;
-import org.apache.flink.streaming.util.serialization.JavaDefaultStringSchema;
-import org.apache.flink.util.Collector;
-import org.apache.flink.util.StringUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import kafka.server.KafkaConfig;
-import kafka.server.KafkaServer;
-import scala.collection.Seq;
-
-/**
- * Code in this test is based on the following GitHub repository:
- * (as per commit bc6b2b2d5f6424d5f377aa6c0871e82a956462ef)
- * <p/>
- * https://github.com/sakserv/hadoop-mini-clusters (ASL licensed)
- */
-
-public class KafkaITCase {
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaITCase.class);
- private static final int NUMBER_OF_KAFKA_SERVERS = 3;
-
- private static int zkPort;
- private static String kafkaHost;
-
- private static String zookeeperConnectionString;
-
- @ClassRule
- public static TemporaryFolder tempFolder = new TemporaryFolder();
- public static File tmpZkDir;
- public static List<File> tmpKafkaDirs;
-
- private static TestingServer zookeeper;
- private static List<KafkaServer> brokers;
- private static String brokerConnectionStrings = "";
-
- private static ConsumerConfig standardCC;
-
- private static ZkClient zkClient;
-
-
- @BeforeClass
- public static void prepare() throws IOException {
- LOG.info("Starting KafkaITCase.prepare()");
- tmpZkDir = tempFolder.newFolder();
-
- tmpKafkaDirs = new ArrayList<File>(NUMBER_OF_KAFKA_SERVERS);
- for (int i = 0; i < NUMBER_OF_KAFKA_SERVERS; i++) {
- tmpKafkaDirs.add(tempFolder.newFolder());
- }
-
- kafkaHost = InetAddress.getLocalHost().getHostName();
- zkPort = NetUtils.getAvailablePort();
- zookeeperConnectionString = "localhost:" + zkPort;
-
- zookeeper = null;
- brokers = null;
-
- try {
- LOG.info("Starting Zookeeper");
- zookeeper = getZookeeper();
- LOG.info("Starting KafkaServer");
- brokers = new ArrayList<KafkaServer>(NUMBER_OF_KAFKA_SERVERS);
- for (int i = 0; i < NUMBER_OF_KAFKA_SERVERS; i++) {
- brokers.add(getKafkaServer(i, tmpKafkaDirs.get(i)));
- SocketServer socketServer = brokers.get(i).socketServer();
- String host = "localhost";
- if(socketServer.host() != null) {
- host = socketServer.host();
- }
- brokerConnectionStrings += host+":"+socketServer.port()+",";
- }
-
- LOG.info("ZK and KafkaServer started.");
- } catch (Throwable t) {
- LOG.warn("Test failed with exception", t);
- Assert.fail("Test failed with: " + t.getMessage());
- }
-
- Properties cProps = new Properties();
- cProps.setProperty("zookeeper.connect", zookeeperConnectionString);
- cProps.setProperty("group.id", "flink-tests");
- cProps.setProperty("auto.commit.enable", "false");
-
- cProps.setProperty("auto.offset.reset", "smallest"); // read from the beginning.
-
- standardCC = new ConsumerConfig(cProps);
-
- zkClient = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(), standardCC.zkConnectionTimeoutMs(), new PersistentKafkaSource.KafkaZKStringSerializer());
- }
-
- @AfterClass
- public static void shutDownServices() {
- LOG.info("Shutting down all services");
- for (KafkaServer broker : brokers) {
- if (broker != null) {
- broker.shutdown();
- }
- }
- if (zookeeper != null) {
- try {
- zookeeper.stop();
- } catch (IOException e) {
- LOG.warn("ZK.stop() failed", e);
- }
- }
- zkClient.close();
- }
-
-
- @Test
- public void testOffsetManipulation() {
- ZkClient zk = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(), standardCC.zkConnectionTimeoutMs(), new PersistentKafkaSource.KafkaZKStringSerializer());
-
- final String topicName = "testOffsetManipulation";
-
- // create topic
- Properties topicConfig = new Properties();
- LOG.info("Creating topic {}", topicName);
- AdminUtils.createTopic(zk, topicName, 3, 2, topicConfig);
-
- PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 0, 1337);
-
- Assert.assertEquals(1337L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 0));
-
- zk.close();
- }
- /**
- * We want to use the High level java consumer API but manage the offset in Zookeeper manually.
- *
- */
- @Test
- public void testPersistentSourceWithOffsetUpdates() throws Exception {
- LOG.info("Starting testPersistentSourceWithOffsetUpdates()");
-
- ZkClient zk = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(), standardCC.zkConnectionTimeoutMs(), new PersistentKafkaSource.KafkaZKStringSerializer());
-
- final String topicName = "testOffsetHacking";
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(3);
- env.getConfig().disableSysoutLogging();
- env.enableCheckpointing(50);
- env.setNumberOfExecutionRetries(0);
-
- // create topic
- Properties topicConfig = new Properties();
- LOG.info("Creating topic {}", topicName);
- AdminUtils.createTopic(zk, topicName, 3, 2, topicConfig);
-
- // write a sequence from 0 to 99 to each of the three partitions.
- writeSequence(env, topicName, 0, 99);
-
-
- readSequence(env, standardCC, topicName, 0, 100, 300);
-
- // check offsets to be set at least higher than 50.
- // correctly, we would expect them to be set to 99, but right now there is no way of stopping a topology once all pending
- // checkpoints have been committed.
- // To work around that limitation, the persistent kafka consumer is throtteled with a thread.sleep().
- long o1 = PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 0);
- long o2 = PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 1);
- long o3 = PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 2);
- Assert.assertTrue("The offset seems incorrect, got "+o1, o1 > 50L);
- Assert.assertTrue("The offset seems incorrect, got "+o2, o2 > 50L);
- Assert.assertTrue("The offset seems incorrect, got "+o3, o3 > 50L);
- /** Once we have proper shutdown of streaming jobs, enable these tests
- Assert.assertEquals("The offset seems incorrect", 99L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 0));
- Assert.assertEquals("The offset seems incorrect", 99L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 1));
- Assert.assertEquals("The offset seems incorrect", 99L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 2));*/
-
-
- LOG.info("Manipulating offsets");
- // set the offset to 25, 50, and 75 for the three partitions
- PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 0, 50);
- PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 1, 50);
- PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 2, 50);
-
- // create new env
- env = StreamExecutionEnvironment.createLocalEnvironment(3);
- env.getConfig().disableSysoutLogging();
- readSequence(env, standardCC, topicName, 50, 50, 150);
-
- zk.close();
-
- LOG.info("Finished testPersistentSourceWithOffsetUpdates()");
- }
-
- private void readSequence(StreamExecutionEnvironment env, ConsumerConfig cc, final String topicName, final int valuesStartFrom, final int valuesCount, final int finalCount) throws Exception {
- LOG.info("Reading sequence for verification until final count {}", finalCount);
- DataStream<Tuple2<Integer, Integer>> source = env.addSource(
- new PersistentKafkaSource<Tuple2<Integer, Integer>>(topicName, new Utils.TypeInformationSerializationSchema<Tuple2<Integer, Integer>>(new Tuple2<Integer, Integer>(1,1), env.getConfig()), cc)
- )
- //add a sleeper mapper. Since there is no good way of "shutting down" a running topology, we have
- // to play this trick. The problem is that we have to wait until all checkpoints are confirmed
- .map(new MapFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>>() {
- @Override
- public Tuple2<Integer, Integer> map(Tuple2<Integer, Integer> value) throws Exception {
- Thread.sleep(150);
- return value;
- }
- }).setParallelism(3);
-
- // verify data
- DataStream<Integer> validIndexes = source.flatMap(new RichFlatMapFunction<Tuple2<Integer, Integer>, Integer>() {
- int[] values = new int[valuesCount];
- int count = 0;
-
- @Override
- public void flatMap(Tuple2<Integer, Integer> value, Collector<Integer> out) throws Exception {
- values[value.f1 - valuesStartFrom]++;
- count++;
-
- LOG.info("Reader " + getRuntimeContext().getIndexOfThisSubtask() + " got " + value + " count=" + count + "/" + finalCount);
- // verify if we've seen everything
-
- if (count == finalCount) {
- LOG.info("Received all values");
- for (int i = 0; i < values.length; i++) {
- int v = values[i];
- if (v != 3) {
- LOG.warn("Test is going to fail");
- printTopic(topicName, valuesCount, this.getRuntimeContext().getExecutionConfig());
- throw new RuntimeException("Expected v to be 3, but was " + v + " on element " + i + " array=" + Arrays.toString(values));
- }
- }
- // test has passed
- throw new SuccessException();
- }
- }
-
- }).setParallelism(1);
-
- tryExecute(env, "Read data from Kafka");
-
- LOG.info("Successfully read sequence for verification");
- }
-
-
-
- private void writeSequence(StreamExecutionEnvironment env, String topicName, final int from, final int to) throws Exception {
- LOG.info("Writing sequence from {} to {} to topic {}", from, to, topicName);
-
- DataStream<Tuple2<Integer, Integer>> stream = env.addSource(new RichParallelSourceFunction<Tuple2<Integer, Integer>>() {
- private static final long serialVersionUID = 1L;
- int cnt = from;
- int partition;
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- partition = getRuntimeContext().getIndexOfThisSubtask();
-
- }
-
- @Override
- public boolean reachedEnd() throws Exception {
- return cnt > to;
- }
-
- @Override
- public Tuple2<Integer, Integer> next() throws Exception {
- LOG.info("Writing " + cnt + " to partition " + partition);
- Tuple2<Integer, Integer> result = new Tuple2<Integer, Integer>(getRuntimeContext().getIndexOfThisSubtask(), cnt);
- cnt++;
- return result;
- }
- }).setParallelism(3);
-
- stream.addSink(new KafkaSink<Tuple2<Integer, Integer>>(brokerConnectionStrings,
- topicName,
- new Utils.TypeInformationSerializationSchema<Tuple2<Integer, Integer>>(new Tuple2<Integer, Integer>(1, 1), env.getConfig()),
- new T2Partitioner()
- )).setParallelism(3);
- env.execute("Write sequence from " + from + " to " + to + " to topic " + topicName);
- LOG.info("Finished writing sequence");
- }
-
- private static class T2Partitioner implements SerializableKafkaPartitioner {
- @Override
- public int partition(Object key, int numPartitions) {
- if(numPartitions != 3) {
- throw new IllegalArgumentException("Expected three partitions");
- }
- Tuple2<Integer, Integer> element = (Tuple2<Integer, Integer>) key;
- return element.f0;
- }
- }
-
-
- @Test
- public void regularKafkaSourceTest() throws Exception {
- LOG.info("Starting KafkaITCase.regularKafkaSourceTest()");
-
- String topic = "regularKafkaSourceTestTopic";
- createTestTopic(topic, 1, 1);
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
- // add consuming topology:
- DataStreamSource<Tuple2<Long, String>> consuming = env.addSource(
- new KafkaSource<Tuple2<Long, String>>(zookeeperConnectionString, topic, "myFlinkGroup", new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()), 5000));
- consuming.addSink(new SinkFunction<Tuple2<Long, String>>() {
- int elCnt = 0;
- int start = -1;
- BitSet validator = new BitSet(101);
-
- @Override
- public void invoke(Tuple2<Long, String> value) throws Exception {
- LOG.debug("Got value = " + value);
- String[] sp = value.f1.split("-");
- int v = Integer.parseInt(sp[1]);
-
- assertEquals(value.f0 - 1000, (long) v);
-
- if (start == -1) {
- start = v;
- }
- Assert.assertFalse("Received tuple twice", validator.get(v - start));
- validator.set(v - start);
- elCnt++;
- if (elCnt == 100) {
- // check if everything in the bitset is set to true
- int nc;
- if ((nc = validator.nextClearBit(0)) != 100) {
- throw new RuntimeException("The bitset was not set to 1 on all elements. Next clear:" + nc + " Set: " + validator);
- }
- throw new SuccessException();
- }
- }
- });
-
- // add producing topology
- DataStream<Tuple2<Long, String>> stream = env.addSource(new SourceFunction<Tuple2<Long, String>>() {
- private static final long serialVersionUID = 1L;
- int cnt = 0;
-
- @Override
- public boolean reachedEnd() throws Exception {
- return false;
- }
-
- @Override
- public Tuple2<Long, String> next() throws Exception {
- Thread.sleep(100);
- return new Tuple2<Long, String>(1000L + cnt, "kafka-" + cnt++);
- }
- });
- stream.addSink(new KafkaSink<Tuple2<Long, String>>(brokerConnectionStrings, topic, new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig())));
-
- tryExecute(env, "regular kafka source test");
-
- LOG.info("Finished KafkaITCase.regularKafkaSourceTest()");
- }
-
- @Test
- public void tupleTestTopology() throws Exception {
- LOG.info("Starting KafkaITCase.tupleTestTopology()");
-
- String topic = "tupleTestTopic";
- createTestTopic(topic, 1, 1);
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- // add consuming topology:
- DataStreamSource<Tuple2<Long, String>> consuming = env.addSource(
- new PersistentKafkaSource<Tuple2<Long, String>>(topic,
- new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()),
- standardCC
- ));
- consuming.addSink(new RichSinkFunction<Tuple2<Long, String>>() {
- int elCnt = 0;
- int start = -1;
- BitSet validator = new BitSet(101);
-
- @Override
- public void invoke(Tuple2<Long, String> value) throws Exception {
- LOG.info("Got value " + value);
- String[] sp = value.f1.split("-");
- int v = Integer.parseInt(sp[1]);
-
- assertEquals(value.f0 - 1000, (long) v);
-
- if (start == -1) {
- start = v;
- }
- Assert.assertFalse("Received tuple twice", validator.get(v - start));
- validator.set(v - start);
- elCnt++;
- if (elCnt == 100) {
- // check if everything in the bitset is set to true
- int nc;
- if ((nc = validator.nextClearBit(0)) != 100) {
- throw new RuntimeException("The bitset was not set to 1 on all elements. Next clear:" + nc + " Set: " + validator);
- }
- throw new SuccessException();
- }
- }
-
- @Override
- public void close() throws Exception {
- super.close();
- Assert.assertTrue("No element received", elCnt > 0);
- }
- });
-
- // add producing topology
- DataStream<Tuple2<Long, String>> stream = env.addSource(new SourceFunction<Tuple2<Long, String>>() {
- private static final long serialVersionUID = 1L;
- int cnt = 0;
-
- @Override
- public boolean reachedEnd() throws Exception {
- return false;
- }
-
- @Override
- public Tuple2<Long, String> next() throws Exception {
- Thread.sleep(100);
- return new Tuple2<Long, String>(1000L + cnt, "kafka-" + cnt++);
- }
- });
- stream.addSink(new KafkaSink<Tuple2<Long, String>>(brokerConnectionStrings, topic, new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig())));
-
- tryExecute(env, "tupletesttopology");
-
- LOG.info("Finished KafkaITCase.tupleTestTopology()");
- }
-
- /**
- * Test Flink's Kafka integration also with very big records (30MB)
- *
- * see http://stackoverflow.com/questions/21020347/kafka-sending-a-15mb-message
- *
- * @throws Exception
- */
- @Test
- public void bigRecordTestTopology() throws Exception {
-
- LOG.info("Starting KafkaITCase.bigRecordTestTopology()");
-
- String topic = "bigRecordTestTopic";
- createTestTopic(topic, 1, 1);
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- // add consuming topology:
- Utils.TypeInformationSerializationSchema<Tuple2<Long, byte[]>> serSchema = new Utils.TypeInformationSerializationSchema<Tuple2<Long, byte[]>>(new Tuple2<Long, byte[]>(0L, new byte[]{0}), env.getConfig());
- Properties consumerProps = new Properties();
- consumerProps.setProperty("fetch.message.max.bytes", Integer.toString(1024 * 1024 * 30));
- consumerProps.setProperty("zookeeper.connect", zookeeperConnectionString);
- consumerProps.setProperty("group.id", "test");
- consumerProps.setProperty("auto.commit.enable", "false");
- consumerProps.setProperty("auto.offset.reset", "smallest");
-
- ConsumerConfig cc = new ConsumerConfig(consumerProps);
- DataStreamSource<Tuple2<Long, byte[]>> consuming = env.addSource(
- new PersistentKafkaSource<Tuple2<Long, byte[]>>(topic, serSchema, cc));
-
- consuming.addSink(new SinkFunction<Tuple2<Long, byte[]>>() {
- int elCnt = 0;
-
- @Override
- public void invoke(Tuple2<Long, byte[]> value) throws Exception {
- LOG.info("Received {}", value.f0);
- elCnt++;
- if(value.f0 == -1) {
- // we should have seen 11 elements now.
- if(elCnt == 11) {
- throw new SuccessException();
- } else {
- throw new RuntimeException("There have been "+elCnt+" elements");
- }
- }
- if(elCnt > 10) {
- throw new RuntimeException("More than 10 elements seen: "+elCnt);
- }
- }
- }).setParallelism(1);
-
- // add producing topology
- DataStream<Tuple2<Long, byte[]>> stream = env.addSource(new RichSourceFunction<Tuple2<Long, byte[]>>() {
- private static final long serialVersionUID = 1L;
- boolean running = true;
- long cnt;
- transient Random rnd;
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- cnt = 0;
- rnd = new Random(1337);
-
- }
-
- @Override
- public boolean reachedEnd() throws Exception {
- return cnt > 10;
- }
-
- @Override
- public Tuple2<Long, byte[]> next() throws Exception {
- Thread.sleep(100);
-
- if (cnt < 10) {
- byte[] wl = new byte[Math.abs(rnd.nextInt(1024 * 1024 * 30))];
- Tuple2<Long, byte[]> result = new Tuple2<Long, byte[]>(cnt++, wl);
- LOG.info("Emitted cnt=" + (cnt - 1) + " with byte.length = " + wl.length);
- return result;
-
- } else if (cnt == 10) {
- Tuple2<Long, byte[]> result = new Tuple2<Long, byte[]>(-1L, new byte[]{1});
- cnt++;
- return result;
- } else {
- throw new RuntimeException("Source is exhausted.");
- }
- }
- });
-
- stream.addSink(new KafkaSink<Tuple2<Long, byte[]>>(brokerConnectionStrings, topic,
- new Utils.TypeInformationSerializationSchema<Tuple2<Long, byte[]>>(new Tuple2<Long, byte[]>(0L, new byte[]{0}), env.getConfig()))
- );
-
- tryExecute(env, "big topology test");
-
- LOG.info("Finished KafkaITCase.bigRecordTestTopology()");
- }
-
-
- @Test
- public void customPartitioningTestTopology() throws Exception {
- LOG.info("Starting KafkaITCase.customPartitioningTestTopology()");
-
- String topic = "customPartitioningTestTopic";
-
- createTestTopic(topic, 3, 1);
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- // add consuming topology:
- DataStreamSource<Tuple2<Long, String>> consuming = env.addSource(
- new PersistentKafkaSource<Tuple2<Long, String>>(topic,
- new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()),
- standardCC));
- consuming.addSink(new SinkFunction<Tuple2<Long, String>>() {
- int start = -1;
- BitSet validator = new BitSet(101);
-
- boolean gotPartition1 = false;
- boolean gotPartition2 = false;
- boolean gotPartition3 = false;
-
- @Override
- public void invoke(Tuple2<Long, String> value) throws Exception {
- LOG.debug("Got " + value);
- String[] sp = value.f1.split("-");
- int v = Integer.parseInt(sp[1]);
-
- assertEquals(value.f0 - 1000, (long) v);
-
- switch (v) {
- case 9:
- gotPartition1 = true;
- break;
- case 19:
- gotPartition2 = true;
- break;
- case 99:
- gotPartition3 = true;
- break;
- }
-
- if (start == -1) {
- start = v;
- }
- Assert.assertFalse("Received tuple twice", validator.get(v - start));
- validator.set(v - start);
-
- if (gotPartition1 && gotPartition2 && gotPartition3) {
- // check if everything in the bitset is set to true
- int nc;
- if ((nc = validator.nextClearBit(0)) != 100) {
- throw new RuntimeException("The bitset was not set to 1 on all elements. Next clear:" + nc + " Set: " + validator);
- }
- throw new SuccessException();
- }
- }
- });
-
- // add producing topology
- DataStream<Tuple2<Long, String>> stream = env.addSource(new SourceFunction<Tuple2<Long, String>>() {
- private static final long serialVersionUID = 1L;
- int cnt = 0;
-
- @Override
- public boolean reachedEnd() throws Exception {
- return false;
- }
-
- @Override
- public Tuple2<Long, String> next() throws Exception {
- Thread.sleep(100);
- return new Tuple2<Long, String>(1000L + cnt, "kafka-" + cnt++);
- }
- });
- stream.addSink(new KafkaSink<Tuple2<Long, String>>(brokerConnectionStrings, topic, new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()), new CustomPartitioner()));
-
- tryExecute(env, "custom partitioning test");
-
- LOG.info("Finished KafkaITCase.customPartitioningTestTopology()");
- }
-
- /**
- * This is for a topic with 3 partitions and Tuple2<Long, String>
- */
- private static class CustomPartitioner implements SerializableKafkaPartitioner {
-
- @Override
- public int partition(Object key, int numPartitions) {
-
- @SuppressWarnings("unchecked")
- Tuple2<Long, String> tuple = (Tuple2<Long, String>) key;
- if (tuple.f0 < 10) {
- return 0;
- } else if (tuple.f0 < 20) {
- return 1;
- } else {
- return 2;
- }
- }
- }
-
-
- @Test
- public void simpleTestTopology() throws Exception {
- String topic = "simpleTestTopic";
-
- createTestTopic(topic, 1, 1);
-
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- // add consuming topology:
- DataStreamSource<String> consuming = env.addSource(
- new PersistentKafkaSource<String>(topic, new JavaDefaultStringSchema(), standardCC));
- consuming.addSink(new SinkFunction<String>() {
- int elCnt = 0;
- int start = -1;
- BitSet validator = new BitSet(101);
-
- @Override
- public void invoke(String value) throws Exception {
- LOG.debug("Got " + value);
- String[] sp = value.split("-");
- int v = Integer.parseInt(sp[1]);
- if (start == -1) {
- start = v;
- }
- Assert.assertFalse("Received tuple twice", validator.get(v - start));
- validator.set(v - start);
- elCnt++;
- if (elCnt == 100) {
- // check if everything in the bitset is set to true
- int nc;
- if ((nc = validator.nextClearBit(0)) != 100) {
- throw new RuntimeException("The bitset was not set to 1 on all elements. Next clear:" + nc + " Set: " + validator);
- }
- throw new SuccessException();
- }
- }
- });
-
- // add producing topology
- DataStream<String> stream = env.addSource(new SourceFunction<String>() {
- private static final long serialVersionUID = 1L;
- boolean running = true;
- int cnt = 0;
-
- @Override
- public boolean reachedEnd() throws Exception {
- return false;
- }
-
- @Override
- public String next() throws Exception {
- Thread.sleep(100);
- return "kafka-" + cnt++;
- }
- });
- stream.addSink(new KafkaSink<String>(brokerConnectionStrings, topic, new JavaDefaultStringSchema()));
-
- tryExecute(env, "simpletest");
- }
-
- private static boolean leaderHasShutDown = false;
- private static boolean shutdownKafkaBroker;
-
- @Test(timeout=60000)
- public void brokerFailureTest() throws Exception {
- String topic = "brokerFailureTestTopic";
-
- createTestTopic(topic, 2, 2);
-
- // --------------------------- write data to topic ---------------------
- LOG.info("Writing data to topic {}", topic);
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- DataStream<String> stream = env.addSource(new SourceFunction<String>() {
-
- private int cnt = 0;
-
- @Override
- public boolean reachedEnd() throws Exception {
- return cnt == 200;
- }
-
- @Override
- public String next() throws Exception {
- String msg = "kafka-" + cnt++;
- LOG.info("sending message = "+msg);
-
- if ((cnt - 1) % 20 == 0) {
- LOG.debug("Sending message #{}", cnt - 1);
- }
-
- return msg;
- }
-
- });
- stream.addSink(new KafkaSink<String>(brokerConnectionStrings, topic, new JavaDefaultStringSchema()))
- .setParallelism(1);
-
- tryExecute(env, "broker failure test - writer");
-
- // --------------------------- read and let broker fail ---------------------
-
- LOG.info("Reading data from topic {} and let a broker fail", topic);
- PartitionMetadata firstPart = null;
- do {
- if(firstPart != null) {
- LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
- // not the first try. Sleep a bit
- Thread.sleep(150);
- }
- Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient).partitionsMetadata();
- firstPart = partitionMetadata.head();
- } while(firstPart.errorCode() != 0);
-
- final String leaderToShutDown = firstPart.leader().get().connectionString();
- LOG.info("Leader to shutdown {}", leaderToShutDown);
-
- final Thread brokerShutdown = new Thread(new Runnable() {
- @Override
- public void run() {
- shutdownKafkaBroker = false;
- while (!shutdownKafkaBroker) {
- try {
- Thread.sleep(10);
- } catch (InterruptedException e) {
- LOG.warn("Interruption", e);
- }
- }
-
- for (KafkaServer kafkaServer : brokers) {
- if (leaderToShutDown.equals(kafkaServer.config().advertisedHostName()+ ":"+ kafkaServer.config().advertisedPort())) {
- LOG.info("Killing Kafka Server {}", leaderToShutDown);
- kafkaServer.shutdown();
- leaderHasShutDown = true;
- break;
- }
- }
- }
- });
- brokerShutdown.start();
-
- // add consuming topology:
- DataStreamSource<String> consuming = env.addSource(new PersistentKafkaSource<String>(topic, new JavaDefaultStringSchema(), standardCC));
- consuming.setParallelism(1);
-
- consuming.addSink(new SinkFunction<String>() {
- int elCnt = 0;
- int start = 0;
- int numOfMessagesToBeCorrect = 100;
- int stopAfterMessages = 150;
-
- BitSet validator = new BitSet(numOfMessagesToBeCorrect + 1);
-
- @Override
- public void invoke(String value) throws Exception {
- LOG.info("Got message = " + value + " leader has shut down " + leaderHasShutDown + " el cnt = " + elCnt + " to rec" + numOfMessagesToBeCorrect);
- String[] sp = value.split("-");
- int v = Integer.parseInt(sp[1]);
-
- if (start == -1) {
- start = v;
- }
- int offset = v - start;
- Assert.assertFalse("Received tuple with value " + offset + " twice", validator.get(offset));
- if (v - start < 0 && LOG.isWarnEnabled()) {
- LOG.warn("Not in order: {}", value);
- }
-
- validator.set(offset);
- elCnt++;
- if (elCnt == 20) {
- LOG.info("Asking leading broker to shut down");
- // shut down a Kafka broker
- shutdownKafkaBroker = true;
- }
- if (shutdownKafkaBroker) {
- // we become a bit slower because the shutdown takes some time and we have
- // only a fixed nubmer of elements to read
- Thread.sleep(20);
- }
- if (leaderHasShutDown) { // it only makes sence to check once the shutdown is completed
- if (elCnt >= stopAfterMessages) {
- // check if everything in the bitset is set to true
- int nc;
- if ((nc = validator.nextClearBit(0)) < numOfMessagesToBeCorrect) {
- throw new RuntimeException("The bitset was not set to 1 on all elements to be checked. Next clear:" + nc + " Set: " + validator);
- }
- throw new SuccessException();
- }
- }
- }
- });
- tryExecute(env, "broker failure test - reader");
-
- }
-
- public static void tryExecute(StreamExecutionEnvironment see, String name) throws Exception {
- try {
- see.execute(name);
- } catch (JobExecutionException good) {
- Throwable t = good.getCause();
- int limit = 0;
- while (!(t instanceof SuccessException)) {
- if(t == null) {
- LOG.warn("Test failed with exception", good);
- Assert.fail("Test failed with: " + StringUtils.stringifyException(good));
- }
-
- t = t.getCause();
- if (limit++ == 20) {
- LOG.warn("Test failed with exception", good);
- Assert.fail("Test failed with: " + StringUtils.stringifyException(good));
- }
- }
- }
- }
-
- private void createTestTopic(String topic, int numberOfPartitions, int replicationFactor) {
- // create topic
- Properties topicConfig = new Properties();
- LOG.info("Creating topic {}", topic);
- AdminUtils.createTopic(zkClient, topic, numberOfPartitions, replicationFactor, topicConfig);
- }
-
- private static TestingServer getZookeeper() throws Exception {
- return new TestingServer(zkPort, tmpZkDir);
- }
-
- /**
- * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed)
- */
- private static KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws UnknownHostException {
- Properties kafkaProperties = new Properties();
-
- int kafkaPort = NetUtils.getAvailablePort();
-
- // properties have to be Strings
- kafkaProperties.put("advertised.host.name", kafkaHost);
- kafkaProperties.put("port", Integer.toString(kafkaPort));
- kafkaProperties.put("broker.id", Integer.toString(brokerId));
- kafkaProperties.put("log.dir", tmpFolder.toString());
- kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
- kafkaProperties.put("message.max.bytes", "" + (35 * 1024 * 1024));
- kafkaProperties.put("replica.fetch.max.bytes", "" + (35 * 1024 * 1024));
- KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);
-
- KafkaServer server = new KafkaServer(kafkaConfig, new KafkaLocalSystemTime());
- server.startup();
- return server;
- }
-
- public static class SuccessException extends Exception {
- private static final long serialVersionUID = 1L;
- }
-
-
- // ----------------------- Debugging utilities --------------------
-
- /**
- * Read topic to list, only using Kafka code.
- * @return
- */
- private static List<MessageAndMetadata<byte[], byte[]>> readTopicToList(String topicName, ConsumerConfig config, final int stopAfter) {
- ConsumerConnector consumerConnector = Consumer.createJavaConsumerConnector(config);
- // we request only one stream per consumer instance. Kafka will make sure that each consumer group
- // will see each message only once.
- Map<String,Integer> topicCountMap = Collections.singletonMap(topicName, 1);
- Map<String, List<KafkaStream<byte[], byte[]>>> streams = consumerConnector.createMessageStreams(topicCountMap);
- if(streams.size() != 1) {
- throw new RuntimeException("Expected only one message stream but got "+streams.size());
- }
- List<KafkaStream<byte[], byte[]>> kafkaStreams = streams.get(topicName);
- if(kafkaStreams == null) {
- throw new RuntimeException("Requested stream not available. Available streams: "+streams.toString());
- }
- if(kafkaStreams.size() != 1) {
- throw new RuntimeException("Requested 1 stream from Kafka, bot got "+kafkaStreams.size()+" streams");
- }
- LOG.info("Opening Consumer instance for topic '{}' on group '{}'", topicName, config.groupId());
- ConsumerIterator<byte[], byte[]> iteratorToRead = kafkaStreams.get(0).iterator();
-
- List<MessageAndMetadata<byte[], byte[]>> result = new ArrayList<MessageAndMetadata<byte[], byte[]>>();
- int read = 0;
- while(iteratorToRead.hasNext()) {
- read++;
- result.add(iteratorToRead.next());
- if(read == stopAfter) {
- LOG.info("Read "+read+" elements");
- return result;
- }
- }
- return result;
- }
-
- private static void printTopic(String topicName, ConsumerConfig config, DeserializationSchema deserializationSchema, int stopAfter){
- List<MessageAndMetadata<byte[], byte[]>> contents = readTopicToList(topicName, config, stopAfter);
- LOG.info("Printing contents of topic {} in consumer grouo {}", topicName, config.groupId());
- for(MessageAndMetadata<byte[], byte[]> message: contents) {
- Object out = deserializationSchema.deserialize(message.message());
- LOG.info("Message: partition: {} offset: {} msg: {}", message.partition(), message.offset(), out.toString());
- }
- }
-
- private static void printTopic(String topicName, int elements, ExecutionConfig ec) {
- // write the sequence to log for debugging purposes
- Properties stdProps = standardCC.props().props();
- Properties newProps = new Properties(stdProps);
- newProps.setProperty("group.id", "topic-printer"+UUID.randomUUID().toString());
- newProps.setProperty("auto.offset.reset", "smallest");
- newProps.setProperty("zookeeper.connect", standardCC.zkConnect());
-
- ConsumerConfig printerConfig = new ConsumerConfig(newProps);
- DeserializationSchema deserializer = new Utils.TypeInformationSerializationSchema<Tuple2<Integer, Integer>>(new Tuple2<Integer, Integer>(1,1), ec);
- printTopic(topicName, printerConfig, deserializer, elements);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/util/KafkaLocalSystemTime.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/util/KafkaLocalSystemTime.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/util/KafkaLocalSystemTime.java
deleted file mode 100644
index 18fa46f..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/util/KafkaLocalSystemTime.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.util;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import kafka.utils.Time;
-
-public class KafkaLocalSystemTime implements Time {
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaLocalSystemTime.class);
-
- @Override
- public long milliseconds() {
- return System.currentTimeMillis();
- }
-
- public long nanoseconds() {
- return System.nanoTime();
- }
-
- @Override
- public void sleep(long ms) {
- try {
- Thread.sleep(ms);
- } catch (InterruptedException e) {
- LOG.warn("Interruption", e);
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
deleted file mode 100644
index 9ede613..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
+++ /dev/null
@@ -1,27 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-log4j.rootLogger=INFO, testlogger
-
-log4j.appender.testlogger=org.apache.log4j.ConsoleAppender
-log4j.appender.testlogger.target = System.err
-log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout
-log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
-
-# suppress the irrelevant (wrong) warnings from the netty channel handler
-log4j.logger.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, testlogger
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml
deleted file mode 100644
index 45b3b92..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/logback-test.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
- </encoder>
- </appender>
-
- <root level="WARN">
- <appender-ref ref="STDOUT"/>
- </root>
- <logger name="org.apache.flink.streaming" level="WARN"/>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/ConnectorSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/ConnectorSource.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/ConnectorSource.java
new file mode 100644
index 0000000..9f59dce
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/ConnectorSource.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.functions.source;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+
+public abstract class ConnectorSource<OUT> extends RichParallelSourceFunction<OUT> implements ResultTypeQueryable<OUT>{
+
+ private static final long serialVersionUID = 1L;
+ protected DeserializationSchema<OUT> schema;
+
+ public ConnectorSource(DeserializationSchema<OUT> schema) {
+ this.schema = schema;
+ }
+
+ @Override
+ public TypeInformation<OUT> getProducedType() {
+ return schema.getProducedType();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/pom.xml b/flink-staging/pom.xml
index 62a2fcd..bc7d101 100644
--- a/flink-staging/pom.xml
+++ b/flink-staging/pom.xml
@@ -65,11 +65,11 @@ under the License.
<module>flink-tachyon</module>
</modules>
</profile>
- <profile>
- <id>include-tez</id>
- <modules>
- <module>flink-tez</module>
- </modules>
- </profile>
+ <profile>
+ <id>include-tez</id>
+ <modules>
+ <module>flink-tez</module>
+ </modules>
+ </profile>
</profiles>
</project>
[2/7] flink git commit: [FLINK-1874] [streaming] Connectors separated
into individual projects
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java
deleted file mode 100644
index 032ed08..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java
+++ /dev/null
@@ -1,338 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.persistent;
-
-import com.google.common.base.Preconditions;
-import kafka.common.TopicAndPartition;
-import kafka.consumer.Consumer;
-import kafka.consumer.ConsumerConfig;
-import kafka.consumer.ConsumerIterator;
-import kafka.consumer.KafkaStream;
-import kafka.javaapi.consumer.ConsumerConnector;
-import kafka.message.MessageAndMetadata;
-import kafka.utils.ZKGroupTopicDirs;
-import kafka.utils.ZkUtils;
-import org.I0Itec.zkclient.ZkClient;
-import org.I0Itec.zkclient.exception.ZkMarshallingError;
-import org.I0Itec.zkclient.serialize.ZkSerializer;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.checkpoint.CheckpointCommitter;
-import org.apache.flink.streaming.api.checkpoint.CheckpointedAsynchronously;
-import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
-import org.apache.flink.streaming.util.serialization.DeserializationSchema;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import scala.Option;
-import scala.collection.JavaConversions;
-import scala.collection.Seq;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-/**
- * Source for reading from Kafka using Flink Streaming Fault Tolerance.
- * This source is updating the committed offset in Zookeeper based on the internal checkpointing of Flink.
- *
- * Note that the autocommit feature of Kafka needs to be disabled for using this source.
- */
-public class PersistentKafkaSource<OUT> extends RichParallelSourceFunction<OUT> implements
- ResultTypeQueryable<OUT>,
- CheckpointCommitter,
- CheckpointedAsynchronously<long[]> {
- private static final Logger LOG = LoggerFactory.getLogger(PersistentKafkaSource.class);
-
- protected transient ConsumerConfig consumerConfig;
- private transient ConsumerIterator<byte[], byte[]> iteratorToRead;
- private transient ConsumerConnector consumer;
-
- private String topicName;
- private DeserializationSchema<OUT> deserializationSchema;
- private boolean running = true;
-
- private transient long[] lastOffsets;
- private transient ZkClient zkClient;
- private transient long[] commitedOffsets; // maintain committed offsets, to avoid committing the same over and over again.
-
- // We set this in reachedEnd to carry it over to next()
- private OUT nextElement = null;
-
- /**
- *
- * For the @param consumerConfig, specify at least the "groupid" and "zookeeper.connect" string.
- * The config will be passed into the Kafka High Level Consumer.
- * For a full list of possible values, check this out: https://kafka.apache.org/documentation.html#consumerconfigs
- */
- public PersistentKafkaSource(String topicName, DeserializationSchema<OUT> deserializationSchema, ConsumerConfig consumerConfig) {
- Preconditions.checkNotNull(topicName);
- Preconditions.checkNotNull(deserializationSchema);
- Preconditions.checkNotNull(consumerConfig);
-
- this.topicName = topicName;
- this.deserializationSchema = deserializationSchema;
- this.consumerConfig = consumerConfig;
- if(consumerConfig.autoCommitEnable()) {
- throw new IllegalArgumentException("'auto.commit.enable' is set to 'true'. " +
- "This source can only be used with auto commit disabled because the " +
- "source is committing to zookeeper by itself (not using the KafkaConsumer).");
- }
- if(!consumerConfig.offsetsStorage().equals("zookeeper")) {
- // we can currently only commit to ZK.
- throw new IllegalArgumentException("The 'offsets.storage' has to be set to 'zookeeper' for this Source to work reliably");
- }
- }
-
- // ---------------------- ParallelSourceFunction Lifecycle -----------------
-
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- ConsumerConnector consumer = Consumer.createJavaConsumerConnector(this.consumerConfig);
- // we request only one stream per consumer instance. Kafka will make sure that each consumer group
- // will see each message only once.
- Map<String,Integer> topicCountMap = Collections.singletonMap(topicName, 1);
- Map<String, List<KafkaStream<byte[], byte[]>>> streams = consumer.createMessageStreams(topicCountMap);
- if(streams.size() != 1) {
- throw new RuntimeException("Expected only one message stream but got "+streams.size());
- }
- List<KafkaStream<byte[], byte[]>> kafkaStreams = streams.get(topicName);
- if(kafkaStreams == null) {
- throw new RuntimeException("Requested stream not available. Available streams: "+streams.toString());
- }
- if(kafkaStreams.size() != 1) {
- throw new RuntimeException("Requested 1 stream from Kafka, bot got "+kafkaStreams.size()+" streams");
- }
- LOG.info("Opening Consumer instance for topic '{}' on group '{}'", topicName, consumerConfig.groupId());
- this.iteratorToRead = kafkaStreams.get(0).iterator();
- this.consumer = consumer;
-
- zkClient = new ZkClient(consumerConfig.zkConnect(),
- consumerConfig.zkSessionTimeoutMs(),
- consumerConfig.zkConnectionTimeoutMs(),
- new KafkaZKStringSerializer());
-
- // most likely the number of offsets we're going to store here will be lower than the number of partitions.
- int numPartitions = getNumberOfPartitions();
- LOG.debug("The topic {} has {} partitions", topicName, numPartitions);
- this.lastOffsets = new long[numPartitions];
- this.commitedOffsets = new long[numPartitions];
- Arrays.fill(this.lastOffsets, -1);
- Arrays.fill(this.commitedOffsets, 0); // just to make it clear
-
- nextElement = null;
- }
-
- @Override
- public boolean reachedEnd() throws Exception {
- if (nextElement != null) {
- return false;
- }
-
- while (iteratorToRead.hasNext()) {
- MessageAndMetadata<byte[], byte[]> message = iteratorToRead.next();
- if(lastOffsets[message.partition()] >= message.offset()) {
- LOG.info("Skipping message with offset {} from partition {}", message.offset(), message.partition());
- continue;
- }
- lastOffsets[message.partition()] = message.offset();
-
- OUT out = deserializationSchema.deserialize(message.message());
- if (deserializationSchema.isEndOfStream(out)) {
- LOG.info("DeserializationSchema signaled end of stream for this source");
- break;
- }
-
- nextElement = out;
- if (LOG.isTraceEnabled()) {
- LOG.trace("Processed record with offset {} from partition {}", message.offset(), message.partition());
- }
- break;
- }
-
- return nextElement == null;
- }
-
- @Override
- public OUT next() throws Exception {
- if (!reachedEnd()) {
- OUT result = nextElement;
- nextElement = null;
- return result;
- } else {
- throw new RuntimeException("Source exhausted");
- }
- }
-
- @Override
- public void close() {
- LOG.info("Closing Kafka consumer");
- this.consumer.shutdown();
- zkClient.close();
- }
-
-
- // ---------------------- State / Checkpoint handling -----------------
- // this source is keeping the partition offsets in Zookeeper
-
- private Map<Long, long[]> pendingCheckpoints = new HashMap<Long, long[]>();
-
- @Override
- public long[] snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
- if(lastOffsets == null) {
- LOG.warn("State snapshot requested on not yet opened source. Returning null");
- return null;
- }
- LOG.info("Snapshotting state. Offsets: {}, checkpoint id {}, timestamp {}", Arrays.toString(lastOffsets), checkpointId, checkpointTimestamp);
-
- long[] currentOffsets = Arrays.copyOf(lastOffsets, lastOffsets.length);
- pendingCheckpoints.put(checkpointId, currentOffsets);
- return currentOffsets;
- }
-
- @Override
- public void restoreState(long[] state) {
- // we maintain the offsets in Kafka, so nothing to do.
- }
-
-
- /**
- * Notification on completed checkpoints
- * @param checkpointId The ID of the checkpoint that has been completed.
- */
- @Override
- public void commitCheckpoint(long checkpointId) {
- LOG.info("Commit checkpoint {}", checkpointId);
- long[] checkpointOffsets = pendingCheckpoints.remove(checkpointId);
- if(checkpointOffsets == null) {
- LOG.warn("Unable to find pending checkpoint for id {}", checkpointId);
- return;
- }
- LOG.info("Got corresponding offsets {}", Arrays.toString(checkpointOffsets));
-
- for(int partition = 0; partition < checkpointOffsets.length; partition++) {
- long offset = checkpointOffsets[partition];
- if(offset != -1) {
- setOffset(partition, offset);
- }
- }
- }
-
- // --------------------- Zookeeper / Offset handling -----------------------------
-
- private int getNumberOfPartitions() {
- scala.collection.immutable.List<String> scalaSeq = JavaConversions.asScalaBuffer(Collections.singletonList(topicName)).toList();
- scala.collection.mutable.Map<String, Seq<Object>> list = ZkUtils.getPartitionsForTopics(zkClient, scalaSeq);
- Option<Seq<Object>> topicOption = list.get(topicName);
- if(topicOption.isEmpty()) {
- throw new IllegalArgumentException("Unable to get number of partitions for topic "+topicName+" from "+list.toString());
- }
- Seq<Object> topic = topicOption.get();
- return topic.size();
- }
-
- protected void setOffset(int partition, long offset) {
- if(commitedOffsets[partition] < offset) {
- setOffset(zkClient, consumerConfig.groupId(), topicName, partition, offset);
- commitedOffsets[partition] = offset;
- } else {
- LOG.debug("Ignoring offset {} for partition {} because it is already committed", offset, partition);
- }
- }
-
-
-
- // the following two methods are static to allow access from the outside as well (Testcases)
-
- /**
- * This method's code is based on ZookeeperConsumerConnector.commitOffsetToZooKeeper()
- */
- public static void setOffset(ZkClient zkClient, String groupId, String topic, int partition, long offset) {
- LOG.info("Setting offset for partition {} of topic {} in group {} to offset {}", partition, topic, groupId, offset);
- TopicAndPartition tap = new TopicAndPartition(topic, partition);
- ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, tap.topic());
- ZkUtils.updatePersistentPath(zkClient, topicDirs.consumerOffsetDir() + "/" + tap.partition(), Long.toString(offset));
- }
-
- public static long getOffset(ZkClient zkClient, String groupId, String topic, int partition) {
- TopicAndPartition tap = new TopicAndPartition(topic, partition);
- ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, tap.topic());
- scala.Tuple2<String, Stat> data = ZkUtils.readData(zkClient, topicDirs.consumerOffsetDir() + "/" + tap.partition());
- return Long.valueOf(data._1());
- }
-
-
- // ---------------------- (Java)Serialization methods for the consumerConfig -----------------
-
- private void writeObject(ObjectOutputStream out)
- throws IOException, ClassNotFoundException {
- out.defaultWriteObject();
- out.writeObject(consumerConfig.props().props());
- }
-
- private void readObject(ObjectInputStream in)
- throws IOException, ClassNotFoundException {
- in.defaultReadObject();
- Properties props = (Properties) in.readObject();
- consumerConfig = new ConsumerConfig(props);
- }
-
-
- @Override
- public TypeInformation<OUT> getProducedType() {
- return deserializationSchema.getProducedType();
- }
-
-
- // ---------------------- Zookeeper Serializer copied from Kafka (because it has private access there) -----------------
-
- public static class KafkaZKStringSerializer implements ZkSerializer {
-
- @Override
- public byte[] serialize(Object data) throws ZkMarshallingError {
- try {
- return ((String) data).getBytes("UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public Object deserialize(byte[] bytes) throws ZkMarshallingError {
- if (bytes == null) {
- return null;
- } else {
- try {
- return new String(bytes, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/KafkaConstantPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/KafkaConstantPartitioner.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/KafkaConstantPartitioner.java
deleted file mode 100644
index 661d0bd..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/KafkaConstantPartitioner.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.partitioner;
-
-public class KafkaConstantPartitioner implements SerializableKafkaPartitioner {
-
- private static final long serialVersionUID = 1L;
- private int partition;
-
- public KafkaConstantPartitioner(int partition) {
- this.partition = partition;
- }
-
- @Override
- public int partition(Object value, int numberOfPartitions) {
- return partition;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/SerializableKafkaPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/SerializableKafkaPartitioner.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/SerializableKafkaPartitioner.java
deleted file mode 100644
index 77a774e..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/SerializableKafkaPartitioner.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.streaming.connectors.kafka.partitioner;
-
-import kafka.producer.Partitioner;
-import java.io.Serializable;
-
-public interface SerializableKafkaPartitioner extends Serializable, Partitioner {
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
deleted file mode 100644
index fa729d6..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.rabbitmq;
-
-import java.io.IOException;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
-import org.apache.flink.streaming.util.serialization.SerializationSchema;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.Connection;
-import com.rabbitmq.client.ConnectionFactory;
-
-public class RMQSink<IN> extends RichSinkFunction<IN> {
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(RMQSink.class);
-
- private String QUEUE_NAME;
- private String HOST_NAME;
- private transient ConnectionFactory factory;
- private transient Connection connection;
- private transient Channel channel;
- private SerializationSchema<IN, byte[]> schema;
-
- public RMQSink(String HOST_NAME, String QUEUE_NAME, SerializationSchema<IN, byte[]> schema) {
- this.HOST_NAME = HOST_NAME;
- this.QUEUE_NAME = QUEUE_NAME;
- this.schema = schema;
- }
-
- /**
- * Initializes the connection to RMQ.
- */
- public void initializeConnection() {
- factory = new ConnectionFactory();
- factory.setHost(HOST_NAME);
- try {
- connection = factory.newConnection();
- channel = connection.createChannel();
- channel.queueDeclare(QUEUE_NAME, false, false, false, null);
-
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Called when new data arrives to the sink, and forwards it to RMQ.
- *
- * @param value
- * The incoming data
- */
- @Override
- public void invoke(IN value) {
- try {
- byte[] msg = schema.serialize(value);
-
- channel.basicPublish("", QUEUE_NAME, null, msg);
-
- } catch (IOException e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Cannot send RMQ message {} at {}", QUEUE_NAME, HOST_NAME);
- }
- }
-
- }
-
- /**
- * Closes the connection.
- */
- private void closeChannel() {
- try {
- channel.close();
- connection.close();
- } catch (IOException e) {
- throw new RuntimeException("Error while closing RMQ connection with " + QUEUE_NAME
- + " at " + HOST_NAME, e);
- }
-
- }
-
- @Override
- public void open(Configuration config) {
- initializeConnection();
- }
-
- @Override
- public void close() {
- closeChannel();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
deleted file mode 100644
index bc087f8..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.rabbitmq;
-
-import java.io.IOException;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.connectors.ConnectorSource;
-import org.apache.flink.streaming.util.serialization.DeserializationSchema;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.Connection;
-import com.rabbitmq.client.ConnectionFactory;
-import com.rabbitmq.client.QueueingConsumer;
-
-public class RMQSource<OUT> extends ConnectorSource<OUT> {
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(RMQSource.class);
-
- private final String QUEUE_NAME;
- private final String HOST_NAME;
-
- private transient ConnectionFactory factory;
- private transient Connection connection;
- private transient Channel channel;
- private transient QueueingConsumer consumer;
- private transient QueueingConsumer.Delivery delivery;
-
- private volatile boolean isRunning = false;
-
- OUT out;
-
- public RMQSource(String HOST_NAME, String QUEUE_NAME,
- DeserializationSchema<OUT> deserializationSchema) {
- super(deserializationSchema);
- this.HOST_NAME = HOST_NAME;
- this.QUEUE_NAME = QUEUE_NAME;
- }
-
- /**
- * Initializes the connection to RMQ.
- */
- private void initializeConnection() {
- factory = new ConnectionFactory();
- factory.setHost(HOST_NAME);
- try {
- connection = factory.newConnection();
- channel = connection.createChannel();
- channel.queueDeclare(QUEUE_NAME, false, false, false, null);
- consumer = new QueueingConsumer(channel);
- channel.basicConsume(QUEUE_NAME, true, consumer);
- } catch (IOException e) {
- throw new RuntimeException("Cannot create RMQ connection with " + QUEUE_NAME + " at "
- + HOST_NAME, e);
- }
- }
-
- @Override
- public void open(Configuration config) throws Exception {
- initializeConnection();
- }
-
- @Override
- public void close() throws Exception {
- super.close();
- try {
- connection.close();
- } catch (IOException e) {
- throw new RuntimeException("Error while closing RMQ connection with " + QUEUE_NAME
- + " at " + HOST_NAME, e);
- }
- }
-
- @Override
- public boolean reachedEnd() throws Exception {
- if (out != null) {
- return true;
- }
- try {
- delivery = consumer.nextDelivery();
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Cannot recieve RMQ message {} at {}", QUEUE_NAME, HOST_NAME);
- }
- }
-
- out = schema.deserialize(delivery.getBody());
- if (schema.isEndOfStream(out)) {
- out = null;
- return false;
- }
- return true;
- }
-
- @Override
- public OUT next() throws Exception {
- if (out != null) {
- OUT result = out;
- out = null;
- return result;
- }
-
- try {
- delivery = consumer.nextDelivery();
- } catch (Exception e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Cannot recieve RMQ message {} at {}", QUEUE_NAME, HOST_NAME);
- }
- }
-
- out = schema.deserialize(delivery.getBody());
- if (schema.isEndOfStream(out)) {
- throw new RuntimeException("RMQ source is at end.");
- }
- OUT result = out;
- out = null;
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
deleted file mode 100644
index 0f06235..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.rabbitmq;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.util.serialization.SerializationSchema;
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
-
-public class RMQTopology {
-
- public static void main(String[] args) throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- @SuppressWarnings("unused")
- DataStream<String> dataStream1 = env.addSource(
- new RMQSource<String>("localhost", "hello", new SimpleStringSchema())).print();
-
- @SuppressWarnings("unused")
- DataStream<String> dataStream2 = env.fromElements("one", "two", "three", "four", "five",
- "q").addSink(
- new RMQSink<String>("localhost", "hello", new StringToByteSerializer()));
-
- env.execute();
- }
-
- public static class StringToByteSerializer implements SerializationSchema<String, byte[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public byte[] serialize(String element) {
- return element.getBytes();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
deleted file mode 100644
index 78e4aa5..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
+++ /dev/null
@@ -1,322 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.twitter;
-
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.util.Properties;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.twitter.hbc.ClientBuilder;
-import com.twitter.hbc.core.Constants;
-import com.twitter.hbc.core.endpoint.StatusesSampleEndpoint;
-import com.twitter.hbc.core.processor.StringDelimitedProcessor;
-import com.twitter.hbc.httpclient.BasicClient;
-import com.twitter.hbc.httpclient.auth.Authentication;
-import com.twitter.hbc.httpclient.auth.OAuth1;
-
-/**
- * Implementation of {@link SourceFunction} specialized to emit tweets from
- * Twitter. It can connect to Twitter Streaming API, collect tweets and
- */
-public class TwitterSource extends RichParallelSourceFunction<String> {
-
- private static final Logger LOG = LoggerFactory.getLogger(TwitterSource.class);
-
- private static final long serialVersionUID = 1L;
- private String authPath;
- private transient BlockingQueue<String> queue;
- private int queueSize = 10000;
- private transient BasicClient client;
- private int waitSec = 5;
-
- private int maxNumberOfTweets;
- private int currentNumberOfTweets;
-
- private String nextElement = null;
-
- private volatile boolean isRunning = false;
-
- /**
- * Create {@link TwitterSource} for streaming
- *
- * @param authPath
- * Location of the properties file containing the required
- * authentication information.
- */
- public TwitterSource(String authPath) {
- this.authPath = authPath;
- maxNumberOfTweets = -1;
- }
-
- /**
- * Create {@link TwitterSource} to collect finite number of tweets
- *
- * @param authPath
- * Location of the properties file containing the required
- * authentication information.
- * @param numberOfTweets
- *
- */
- public TwitterSource(String authPath, int numberOfTweets) {
- this.authPath = authPath;
- this.maxNumberOfTweets = numberOfTweets;
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- initializeConnection();
- currentNumberOfTweets = 0;
- }
-
- /**
- * Initialize Hosebird Client to be able to consume Twitter's Streaming API
- */
- private void initializeConnection() {
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Initializing Twitter Streaming API connection");
- }
-
- queue = new LinkedBlockingQueue<String>(queueSize);
-
- StatusesSampleEndpoint endpoint = new StatusesSampleEndpoint();
- endpoint.stallWarnings(false);
-
- Authentication auth = authenticate();
-
- initializeClient(endpoint, auth);
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Twitter Streaming API connection established successfully");
- }
- }
-
- private OAuth1 authenticate() {
-
- Properties authenticationProperties = loadAuthenticationProperties();
-
- return new OAuth1(authenticationProperties.getProperty("consumerKey"),
- authenticationProperties.getProperty("consumerSecret"),
- authenticationProperties.getProperty("token"),
- authenticationProperties.getProperty("secret"));
- }
-
- /**
- * Reads the given properties file for the authentication data.
- *
- * @return the authentication data.
- */
- private Properties loadAuthenticationProperties() {
- Properties properties = new Properties();
- try {
- InputStream input = new FileInputStream(authPath);
- properties.load(input);
- input.close();
- } catch (Exception e) {
- throw new RuntimeException("Cannot open .properties file: " + authPath, e);
- }
- return properties;
- }
-
- private void initializeClient(StatusesSampleEndpoint endpoint, Authentication auth) {
-
- client = new ClientBuilder().name("twitterSourceClient").hosts(Constants.STREAM_HOST)
- .endpoint(endpoint).authentication(auth)
- .processor(new StringDelimitedProcessor(queue)).build();
-
- client.connect();
- }
-
- /**
- * Put tweets into output
- *
- * @param collector
- * Collector in which the tweets are collected.
- */
- protected void collectFiniteMessages(Collector<String> collector) {
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Collecting tweets");
- }
-
- for (int i = 0; i < maxNumberOfTweets; i++) {
- collectOneMessage(collector);
- }
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Collecting tweets finished");
- }
- }
-
- /**
- * Put tweets into output
- *
- * @param collector
- * Collector in which the tweets are collected.
- */
- protected void collectMessages(Collector<String> collector) {
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Tweet-stream begins");
- }
-
- while (isRunning) {
- collectOneMessage(collector);
- }
- }
-
- /**
- * Put one tweet into the output.
- *
- * @param collector
- * Collector in which the tweets are collected.
- */
- protected void collectOneMessage(Collector<String> collector) {
- if (client.isDone()) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Client connection closed unexpectedly: {}", client.getExitEvent()
- .getMessage());
- }
- }
-
- try {
- String msg = queue.poll(waitSec, TimeUnit.SECONDS);
- if (msg != null) {
- collector.collect(msg);
- } else {
- if (LOG.isInfoEnabled()) {
- LOG.info("Did not receive a message in {} seconds", waitSec);
- }
- }
- } catch (InterruptedException e) {
- throw new RuntimeException("'Waiting for tweet' thread is interrupted", e);
- }
-
- }
-
- private void closeConnection() {
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Initiating connection close");
- }
-
- client.stop();
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Connection closed successfully");
- }
- }
-
- /**
- * Get the size of the queue in which the tweets are contained temporarily.
- *
- * @return the size of the queue in which the tweets are contained
- * temporarily
- */
- public int getQueueSize() {
- return queueSize;
- }
-
- /**
- * Set the size of the queue in which the tweets are contained temporarily.
- *
- * @param queueSize
- * The desired value.
- */
- public void setQueueSize(int queueSize) {
- this.queueSize = queueSize;
- }
-
- /**
- * This function tells how long TwitterSource waits for the tweets.
- *
- * @return Number of second.
- */
- public int getWaitSec() {
- return waitSec;
- }
-
- /**
- * This function sets how long TwitterSource should wait for the tweets.
- *
- * @param waitSec
- * The desired value.
- */
- public void setWaitSec(int waitSec) {
- this.waitSec = waitSec;
- }
-
- @Override
- public boolean reachedEnd() throws Exception {
- if (currentNumberOfTweets >= maxNumberOfTweets) {
- return false;
- }
-
- if (nextElement != null) {
- return true;
- }
- if (client.isDone()) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Client connection closed unexpectedly: {}", client.getExitEvent()
- .getMessage());
- }
- return false;
- }
-
- try {
- String msg = queue.poll(waitSec, TimeUnit.SECONDS);
- if (msg != null) {
- nextElement = msg;
- return true;
- } else {
- if (LOG.isInfoEnabled()) {
- LOG.info("Did not receive a message in {} seconds", waitSec);
- }
- }
- } catch (InterruptedException e) {
- throw new RuntimeException("'Waiting for tweet' thread is interrupted", e);
- }
- return false;
- }
-
- @Override
- public String next() throws Exception {
- if (nextElement != null) {
- String result = nextElement;
- nextElement = null;
- return result;
- }
- if (reachedEnd()) {
- throw new RuntimeException("Twitter stream end reached.");
- } else {
- String result = nextElement;
- nextElement = null;
- return result;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
deleted file mode 100644
index e500fef..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.twitter;
-
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.sink.SinkFunction;
-import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
-import org.apache.flink.util.Collector;
-import org.apache.sling.commons.json.JSONException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TwitterStreaming {
-
- private static final int PARALLELISM = 1;
- private static final int SOURCE_PARALLELISM = 1;
- private static final int NUMBEROFTWEETS = 100;
-
- private static final Logger LOG = LoggerFactory.getLogger(TwitterStreaming.class);
-
- public static class TwitterSink implements SinkFunction<Tuple5<Long, Integer, String, String, String>> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(Tuple5<Long, Integer, String, String, String> tuple) {
- System.out.println("ID: " + tuple.f0 + " int: " + tuple.f1 + " LANGUAGE: " + tuple.f2);
- System.out.println("NAME: " + tuple.f4);
- System.out.println("TEXT: " + tuple.f3);
- System.out.println("");
- }
-
- }
-
- public static class SelectDataFlatMap extends
- JSONParseFlatMap<String, Tuple5<Long, Integer, String, String, String>> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(String value, Collector<Tuple5<Long, Integer, String, String, String>> out)
- throws Exception {
- try {
- out.collect(new Tuple5<Long, Integer, String, String, String>(
- getLong(value, "id"),
- getInt(value, "entities.hashtags[0].indices[1]"),
- getString(value, "lang"),
- getString(value, "text"),
- getString(value, "user.name")));
- } catch (JSONException e) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Field not found");
- }
- }
- }
- }
-
- public static void main(String[] args) throws Exception {
-
- String path = new String();
-
- if (args != null && args.length == 1) {
- path = args[0];
- } else {
- System.err.println("USAGE:\nTwitterStreaming <pathToPropertiesFile>");
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment
- .createLocalEnvironment(PARALLELISM);
-
- DataStream<String> streamSource = env.addSource(new TwitterSource(path, NUMBEROFTWEETS))
- .setParallelism(SOURCE_PARALLELISM);
-
- DataStream<Tuple5<Long, Integer, String, String, String>> selectedDataStream = streamSource
- .flatMap(new SelectDataFlatMap());
-
- selectedDataStream.addSink(new TwitterSink());
-
- env.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
deleted file mode 100644
index 4bc6df0..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.twitter;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
-import org.apache.flink.util.Collector;
-import org.apache.sling.commons.json.JSONException;
-
-/**
- * This program demonstrate the use of TwitterSource.
- * Its aim is to count the frequency of the languages of tweets
- */
-public class TwitterTopology {
-
- private static final int NUMBEROFTWEETS = 100;
-
- /**
- * FlatMapFunction to determine the language of tweets if possible
- */
- public static class SelectLanguageFlatMap extends
- JSONParseFlatMap<String, String> {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * Select the language from the incoming JSON text
- */
- @Override
- public void flatMap(String value, Collector<String> out) throws Exception {
- try{
- out.collect(getString(value, "lang"));
- }
- catch (JSONException e){
- out.collect("");
- }
- }
-
- }
-
- public static void main(String[] args) throws Exception {
-
- String path = new String();
-
- if (args != null && args.length == 1) {
- path = args[0];
- } else {
- System.err.println("USAGE:\nTwitterLocal <pathToPropertiesFile>");
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<String> streamSource = env.addSource(new TwitterSource(path, NUMBEROFTWEETS));
-
-
- DataStream<Tuple2<String, Integer>> dataStream = streamSource
- .flatMap(new SelectLanguageFlatMap())
- .map(new MapFunction<String, Tuple2<String, Integer>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<String, Integer> map(String value) throws Exception {
- return new Tuple2<String, Integer>(value, 1);
- }
- })
- .groupBy(0)
- .sum(1);
-
- dataStream.print();
-
- env.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
deleted file mode 100644
index b1d4115..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.json;
-
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.apache.flink.streaming.connectors.json.JSONParser;
-import org.apache.sling.commons.json.JSONException;
-import org.apache.sling.commons.json.JSONObject;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class JSONParserTest {
-
- private String jsonText;
- private String searchedField;
-
- public JSONParserTest(String text, String field) {
- jsonText = text;
- searchedField = field;
- }
-
- @Parameters
- public static Collection<Object[]> initParameterList() {
-
- Object[][] parameterList = new Object[][] {
- { "{\"key\":\"value\"}", "key" },
- { "{\"key\":[\"value\"]}", "key[0]" },
- { "{\"key\":[{\"key\":\"value\"}]}", "key[0].key" },
- { "{\"key\":[{\"key\":[{\"key\":\"value\"}]}]}", "key[0].key[0].key"},
- { "{\"key\":[1,[{\"key\":\"value\"}]]}", "key[1][0].key" },
- { "{\"key\":[1,[[\"key\",2,\"value\"]]]}", "key[1][0][2]" },
- { "{\"key\":{\"key\":{\"otherKey\":\"wrongValue\",\"key\":\"value\"},\"otherKey\":\"wrongValue\"},\"otherKey\":\"wrongValue\"}" , "key.key.key"}
- };
-
- return Arrays.asList(parameterList);
- }
-
- @Test
- public void test() {
- try {
- JSONParser parser = new JSONParser(jsonText);
- JSONObject jo = parser.parse(searchedField);
- String expected = "{\"retValue\":\"value\"}";
-
- assertTrue(expected.equals(jo.toString()));
- }
- catch (JSONException e) {
- fail();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
deleted file mode 100644
index 8851086..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.json;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.flink.streaming.connectors.json.JSONParser;
-import org.apache.sling.commons.json.JSONException;
-import org.apache.sling.commons.json.JSONObject;
-import org.junit.Test;
-
-
-public class JSONParserTest2 {
-
- @Test
- public void testGetBooleanFunction() {
- String jsonText = "{\"key\":true}";
- String searchedField = "key";
- try {
- JSONParser parser = new JSONParser(jsonText);
- JSONObject jo = parser.parse(searchedField);
-
- assertTrue(jo.getBoolean("retValue"));
- }
- catch (JSONException e) {
- fail();
- }
- }
-
- @Test
- public void testGetDoubleFunction() {
- double expected = 12345.12345;
- String jsonText = "{\"key\":" + expected + "}";
- String searchedField = "key";
- try {
- JSONParser parser = new JSONParser(jsonText);
- JSONObject jo = parser.parse(searchedField);
-
- assertEquals(expected,jo.getDouble("retValue"),0.000001);
- }
- catch (JSONException e) {
- fail();
- }
- }
-
- @Test
- public void testGetIntFunction() {
- int expected = 15;
- String jsonText = "{\"key\":" + expected + "}";
- String searchedField = "key";
- try {
- JSONParser parser = new JSONParser(jsonText);
- JSONObject jo = parser.parse(searchedField);
-
- assertEquals(expected,jo.getInt("retValue"));
- }
- catch (JSONException e) {
- fail();
- }
- }
-
- @Test
- public void testGetLongFunction() {
- long expected = 111111111111L;
- String jsonText = "{\"key\":" + expected + "}";
- String searchedField = "key";
- try {
- JSONParser parser = new JSONParser(jsonText);
- JSONObject jo = parser.parse(searchedField);
-
- assertEquals(expected,jo.getLong("retValue"));
- }
- catch (JSONException e) {
- fail();
- }
- }
-
-}
[5/7] flink git commit: [FLINK-1874] [streaming] Connectors separated
into individual projects
Posted by mb...@apache.org.
[FLINK-1874] [streaming] Connectors separated into individual projects
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/665bcec7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/665bcec7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/665bcec7
Branch: refs/heads/master
Commit: 665bcec779ef16e9ae2ba535631765d329f81392
Parents: 72828b5
Author: mbalassi <mb...@apache.org>
Authored: Sat May 23 10:50:58 2015 +0200
Committer: mbalassi <mb...@apache.org>
Committed: Thu May 28 14:44:22 2015 +0200
----------------------------------------------------------------------
docs/apis/streaming_guide.md | 30 +-
.../flink-connector-flume/pom.xml | 171 +++
.../streaming/connectors/flume/FlumeSink.java | 141 +++
.../streaming/connectors/flume/FlumeSource.java | 149 +++
.../connectors/flume/FlumeTopology.java | 49 +
.../flink-connector-kafka/pom.xml | 103 ++
.../connectors/kafka/KafkaConsumerExample.java | 58 +
.../connectors/kafka/KafkaProducerExample.java | 80 ++
.../flink/streaming/connectors/kafka/Utils.java | 70 ++
.../connectors/kafka/api/KafkaSink.java | 193 ++++
.../connectors/kafka/api/KafkaSource.java | 219 ++++
.../kafka/api/config/PartitionerWrapper.java | 49 +
.../api/persistent/PersistentKafkaSource.java | 338 ++++++
.../partitioner/KafkaConstantPartitioner.java | 33 +
.../SerializableKafkaPartitioner.java | 24 +
.../flink/streaming/kafka/KafkaITCase.java | 1023 ++++++++++++++++++
.../kafka/util/KafkaLocalSystemTime.java | 48 +
.../src/test/resources/log4j-test.properties | 27 +
.../src/test/resources/logback-test.xml | 30 +
.../flink-connector-rabbitmq/pom.xml | 59 +
.../streaming/connectors/rabbitmq/RMQSink.java | 111 ++
.../connectors/rabbitmq/RMQSource.java | 138 +++
.../connectors/rabbitmq/RMQTopology.java | 52 +
.../flink-connector-twitter/pom.xml | 95 ++
.../connectors/json/JSONParseFlatMap.java | 144 +++
.../streaming/connectors/json/JSONParser.java | 175 +++
.../connectors/twitter/TwitterSource.java | 322 ++++++
.../connectors/twitter/TwitterStreaming.java | 99 ++
.../connectors/twitter/TwitterTopology.java | 92 ++
.../connectors/json/JSONParserTest.java | 74 ++
.../connectors/json/JSONParserTest2.java | 95 ++
.../src/test/resources/log4j-test.properties | 27 +
.../src/test/resources/logback-test.xml | 30 +
.../flink-streaming-connectors/pom.xml | 215 +---
.../streaming/connectors/ConnectorSource.java | 38 -
.../streaming/connectors/flume/FlumeSink.java | 141 ---
.../streaming/connectors/flume/FlumeSource.java | 149 ---
.../connectors/flume/FlumeTopology.java | 49 -
.../connectors/json/JSONParseFlatMap.java | 144 ---
.../streaming/connectors/json/JSONParser.java | 175 ---
.../connectors/kafka/KafkaConsumerExample.java | 58 -
.../connectors/kafka/KafkaProducerExample.java | 80 --
.../flink/streaming/connectors/kafka/Utils.java | 70 --
.../connectors/kafka/api/KafkaSink.java | 193 ----
.../connectors/kafka/api/KafkaSource.java | 219 ----
.../kafka/api/config/PartitionerWrapper.java | 49 -
.../api/persistent/PersistentKafkaSource.java | 338 ------
.../partitioner/KafkaConstantPartitioner.java | 33 -
.../SerializableKafkaPartitioner.java | 24 -
.../streaming/connectors/rabbitmq/RMQSink.java | 111 --
.../connectors/rabbitmq/RMQSource.java | 138 ---
.../connectors/rabbitmq/RMQTopology.java | 52 -
.../connectors/twitter/TwitterSource.java | 322 ------
.../connectors/twitter/TwitterStreaming.java | 99 --
.../connectors/twitter/TwitterTopology.java | 92 --
.../connectors/json/JSONParserTest.java | 74 --
.../connectors/json/JSONParserTest2.java | 95 --
.../streaming/connectors/kafka/KafkaITCase.java | 1023 ------------------
.../kafka/util/KafkaLocalSystemTime.java | 48 -
.../src/test/resources/log4j-test.properties | 27 -
.../src/test/resources/logback-test.xml | 30 -
.../api/functions/source/ConnectorSource.java | 38 +
flink-staging/pom.xml | 12 +-
63 files changed, 4397 insertions(+), 4087 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/docs/apis/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/apis/streaming_guide.md b/docs/apis/streaming_guide.md
index ab23695..6565fbf 100644
--- a/docs/apis/streaming_guide.md
+++ b/docs/apis/streaming_guide.md
@@ -1213,7 +1213,15 @@ To run an application using one of these connectors usually additional third par
### Apache Kafka
-This connector provides access to data streams from [Apache Kafka](https://kafka.apache.org/).
+This connector provides access to data streams from [Apache Kafka](https://kafka.apache.org/). For using this connector add the following dependency to your project:
+
+{% highlight xml %}
+<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-kafka-connector</artifactId>
+ <version>{{site.version }}</version>
+</dependency>
+{% endhighlight %}
#### Installing Apache Kafka
* Follow the instructions from [Kafka's quickstart](https://kafka.apache.org/documentation.html#quickstart) to download the code and launch a server (launching a Zookeeper and a Kafka server is required every time before starting the application).
@@ -1433,7 +1441,15 @@ More on Flume can be found [here](http://flume.apache.org).
### RabbitMQ
-This connector provides access to data streams from [RabbitMQ](http://www.rabbitmq.com/).
+This connector provides access to data streams from [RabbitMQ](http://www.rabbitmq.com/).For using this connector add the following dependency to your project:
+
+{% highlight xml %}
+<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-rabbitmq-connector</artifactId>
+ <version>{{site.version }}</version>
+</dependency>
+{% endhighlight %}
##### Installing RabbitMQ
Follow the instructions from the [RabbitMQ download page](http://www.rabbitmq.com/download.html). After the installation the server automatically starts and the application connecting to RabbitMQ can be launched.
@@ -1497,7 +1513,15 @@ More about RabbitMQ can be found [here](http://www.rabbitmq.com/).
### Twitter Streaming API
-Twitter Streaming API provides opportunity to connect to the stream of tweets made available by Twitter. Flink Streaming comes with a built-in `TwitterSource` class for establishing a connection to this stream.
+Twitter Streaming API provides opportunity to connect to the stream of tweets made available by Twitter. Flink Streaming comes with a built-in `TwitterSource` class for establishing a connection to this stream.For using this connector add the following dependency to your project:
+
+{% highlight xml %}
+<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-twitter-connector</artifactId>
+ <version>{{site.version }}</version>
+</dependency>
+{% endhighlight %}
#### Authentication
In order to connect to Twitter stream the user has to register their program and acquire the necessary information for the authentication. The process is described below.
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/pom.xml
new file mode 100644
index 0000000..9ed9777
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/pom.xml
@@ -0,0 +1,171 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-connectors-parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-connector-flume</artifactId>
+ <name>flink-connector-flume</name>
+
+ <packaging>jar</packaging>
+
+ <!-- Allow users to pass custom connector versions -->
+ <properties>
+ <flume-ng.version>1.5.0</flume-ng.version>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-ng-core</artifactId>
+ <version>${flume-ng.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.tukaani</groupId>
+ <artifactId>xz</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <!-- Override artifactSet configuration to build fat-jar with all dependencies packed. -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <configuration>
+ <artifactSet>
+ <includes combine.children="append">
+ <!-- We include all dependencies that transitively depend on guava -->
+ <include>org.apache.flume:*</include>
+ </includes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
new file mode 100644
index 0000000..50f5770
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.flume;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.flink.streaming.util.serialization.SerializationSchema;
+import org.apache.flume.Event;
+import org.apache.flume.EventDeliveryException;
+import org.apache.flume.FlumeException;
+import org.apache.flume.api.RpcClient;
+import org.apache.flume.api.RpcClientFactory;
+import org.apache.flume.event.EventBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FlumeSink<IN> extends RichSinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(FlumeSink.class);
+
+ private transient FlinkRpcClientFacade client;
+ boolean initDone = false;
+ String host;
+ int port;
+ SerializationSchema<IN, byte[]> schema;
+
+ public FlumeSink(String host, int port, SerializationSchema<IN, byte[]> schema) {
+ this.host = host;
+ this.port = port;
+ this.schema = schema;
+ }
+
+ /**
+ * Receives tuples from the Apache Flink {@link DataStream} and forwards
+ * them to Apache Flume.
+ *
+ * @param value
+ * The tuple arriving from the datastream
+ */
+ @Override
+ public void invoke(IN value) {
+
+ byte[] data = schema.serialize(value);
+ client.sendDataToFlume(data);
+
+ }
+
+ private class FlinkRpcClientFacade {
+ private RpcClient client;
+ private String hostname;
+ private int port;
+
+ /**
+ * Initializes the connection to Apache Flume.
+ *
+ * @param hostname
+ * The host
+ * @param port
+ * The port.
+ */
+ public void init(String hostname, int port) {
+ // Setup the RPC connection
+ this.hostname = hostname;
+ this.port = port;
+ int initCounter = 0;
+ while (true) {
+ if (initCounter >= 90) {
+ throw new RuntimeException("Cannot establish connection with" + port + " at "
+ + host);
+ }
+ try {
+ this.client = RpcClientFactory.getDefaultInstance(hostname, port);
+ } catch (FlumeException e) {
+ // Wait one second if the connection failed before the next
+ // try
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e1) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Interrupted while trying to connect {} at {}", port, host);
+ }
+ }
+ }
+ if (client != null) {
+ break;
+ }
+ initCounter++;
+ }
+ initDone = true;
+ }
+
+ /**
+ * Sends byte arrays as {@link Event} series to Apache Flume.
+ *
+ * @param data
+ * The byte array to send to Apache FLume
+ */
+ public void sendDataToFlume(byte[] data) {
+ Event event = EventBuilder.withBody(data);
+
+ try {
+ client.append(event);
+
+ } catch (EventDeliveryException e) {
+ // clean up and recreate the client
+ client.close();
+ client = null;
+ client = RpcClientFactory.getDefaultInstance(hostname, port);
+ }
+ }
+
+ }
+
+ @Override
+ public void close() {
+ client.client.close();
+ }
+
+ @Override
+ public void open(Configuration config) {
+ client = new FlinkRpcClientFacade();
+ client.init(host, port);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
new file mode 100644
index 0000000..8fecd0f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
@@ -0,0 +1,149 @@
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one or more
+// * contributor license agreements. See the NOTICE file distributed with
+// * this work for additional information regarding copyright ownership.
+// * The ASF licenses this file to You under the Apache License, Version 2.0
+// * (the "License"); you may not use this file except in compliance with
+// * the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.flink.streaming.connectors.flume;
+//
+//import java.util.List;
+//
+//import org.apache.flink.streaming.api.datastream.DataStream;
+//import org.apache.flink.streaming.api.functions.source.ConnectorSource;
+//import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+//import org.apache.flink.util.Collector;
+//import org.apache.flume.Context;
+//import org.apache.flume.channel.ChannelProcessor;
+//import org.apache.flume.source.AvroSource;
+//import org.apache.flume.source.avro.AvroFlumeEvent;
+//import org.apache.flume.source.avro.Status;
+//
+//public class FlumeSource<OUT> extends ConnectorSource<OUT> {
+// private static final long serialVersionUID = 1L;
+//
+// String host;
+// String port;
+// volatile boolean finished = false;
+//
+// private volatile boolean isRunning = false;
+//
+// FlumeSource(String host, int port, DeserializationSchema<OUT> deserializationSchema) {
+// super(deserializationSchema);
+// this.host = host;
+// this.port = Integer.toString(port);
+// }
+//
+// public class MyAvroSource extends AvroSource {
+// Collector<OUT> output;
+//
+// /**
+// * Sends the AvroFlumeEvent from it's argument list to the Apache Flink
+// * {@link DataStream}.
+// *
+// * @param avroEvent
+// * The event that should be sent to the dataStream
+// * @return A {@link Status}.OK message if sending the event was
+// * successful.
+// */
+// @Override
+// public Status append(AvroFlumeEvent avroEvent) {
+// collect(avroEvent);
+// return Status.OK;
+// }
+//
+// /**
+// * Sends the AvroFlumeEvents from it's argument list to the Apache Flink
+// * {@link DataStream}.
+// *
+// * @param events
+// * The events that is sent to the dataStream
+// * @return A Status.OK message if sending the events was successful.
+// */
+// @Override
+// public Status appendBatch(List<AvroFlumeEvent> events) {
+// for (AvroFlumeEvent avroEvent : events) {
+// collect(avroEvent);
+// }
+//
+// return Status.OK;
+// }
+//
+// /**
+// * Deserializes the AvroFlumeEvent before sending it to the Apache Flink
+// * {@link DataStream}.
+// *
+// * @param avroEvent
+// * The event that is sent to the dataStream
+// */
+// private void collect(AvroFlumeEvent avroEvent) {
+// byte[] b = avroEvent.getBody().array();
+// OUT out = FlumeSource.this.schema.deserialize(b);
+//
+// if (schema.isEndOfStream(out)) {
+// FlumeSource.this.finished = true;
+// this.stop();
+// FlumeSource.this.notifyAll();
+// } else {
+// output.collect(out);
+// }
+//
+// }
+//
+// }
+//
+// MyAvroSource avroSource;
+//
+// /**
+// * Configures the AvroSource. Also sets the output so the application can
+// * use it from outside of the invoke function.
+// *
+// * @param output
+// * The output used in the invoke function
+// */
+// public void configureAvroSource(Collector<OUT> output) {
+//
+// avroSource = new MyAvroSource();
+// avroSource.output = output;
+// Context context = new Context();
+// context.put("port", port);
+// context.put("bind", host);
+// avroSource.configure(context);
+// // An instance of a ChannelProcessor is required for configuring the
+// // avroSource although it will not be used in this case.
+// ChannelProcessor cp = new ChannelProcessor(null);
+// avroSource.setChannelProcessor(cp);
+// }
+//
+// /**
+// * Configures the AvroSource and runs until the user calls a close function.
+// *
+// * @param output
+// * The Collector for sending data to the datastream
+// */
+// @Override
+// public void run(Collector<OUT> output) throws Exception {
+// isRunning = true;
+// configureAvroSource(output);
+// avroSource.start();
+// while (!finished && isRunning) {
+// this.wait();
+// }
+// }
+//
+// @Override
+// public void cancel() {
+// isRunning = false;
+// }
+//
+//}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
new file mode 100644
index 0000000..f630bce
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-flume/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
@@ -0,0 +1,49 @@
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one or more
+// * contributor license agreements. See the NOTICE file distributed with
+// * this work for additional information regarding copyright ownership.
+// * The ASF licenses this file to You under the Apache License, Version 2.0
+// * (the "License"); you may not use this file except in compliance with
+// * the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.flink.streaming.connectors.flume;
+//
+//import org.apache.flink.streaming.api.datastream.DataStream;
+//import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+//import org.apache.flink.streaming.util.serialization.SerializationSchema;
+//import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
+//
+//public class FlumeTopology {
+//
+// public static void main(String[] args) throws Exception {
+//
+// StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+//
+// @SuppressWarnings("unused")
+// DataStream<String> dataStream1 = env.addSource(
+// new FlumeSource<String>("localhost", 41414, new SimpleStringSchema())).addSink(
+// new FlumeSink<String>("localhost", 42424, new StringToByteSerializer()));
+//
+// env.execute();
+// }
+//
+// public static class StringToByteSerializer implements SerializationSchema<String, byte[]> {
+//
+// private static final long serialVersionUID = 1L;
+//
+// @Override
+// public byte[] serialize(String element) {
+// return element.getBytes();
+// }
+// }
+//
+//}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml
new file mode 100644
index 0000000..e9c9da2
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml
@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-connectors-parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-connector-kafka</artifactId>
+ <name>flink-connector-kafka</name>
+
+ <packaging>jar</packaging>
+
+ <!-- Allow users to pass custom connector versions -->
+ <properties>
+ <kafka.version>0.8.2.0</kafka.version>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_${scala.binary.version}</artifactId>
+ <version>${kafka.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.sf.jopt-simple</groupId>
+ <artifactId>jopt-simple</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-reflect</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.yammer.metrics</groupId>
+ <artifactId>metrics-annotation</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <version>2.7.1</version>
+ </dependency>
+
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java
new file mode 100644
index 0000000..fe6684d
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.kafka.api.KafkaSource;
+import org.apache.flink.streaming.util.serialization.JavaDefaultStringSchema;
+
+public class KafkaConsumerExample {
+
+ private static String host;
+ private static int port;
+ private static String topic;
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(4);
+
+ DataStream<String> kafkaStream = env
+ .addSource(new KafkaSource<String>(host + ":" + port, topic, new JavaDefaultStringSchema()));
+ kafkaStream.print();
+
+ env.execute();
+ }
+
+ private static boolean parseParameters(String[] args) {
+ if (args.length == 3) {
+ host = args[0];
+ port = Integer.parseInt(args[1]);
+ topic = args[2];
+ return true;
+ } else {
+ System.err.println("Usage: KafkaConsumerExample <host> <port> <topic>");
+ return false;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerExample.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerExample.java
new file mode 100644
index 0000000..4dd5577
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerExample.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.connectors.kafka.api.KafkaSink;
+import org.apache.flink.streaming.util.serialization.JavaDefaultStringSchema;
+
+public class KafkaProducerExample {
+
+ private static String host;
+ private static int port;
+ private static String topic;
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(4);
+
+ @SuppressWarnings({ "unused", "serial" })
+ DataStream<String> stream1 = env.addSource(new SourceFunction<String>() {
+
+ private int index = 0;
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ return index >= 20;
+ }
+
+ @Override
+ public String next() throws Exception {
+ if (index < 20) {
+ String result = "message #" + index;
+ index++;
+ return result;
+ }
+
+ return "q";
+ }
+
+ }).addSink(
+ new KafkaSink<String>(host + ":" + port, topic, new JavaDefaultStringSchema())
+ )
+ .setParallelism(3);
+
+ env.execute();
+ }
+
+ private static boolean parseParameters(String[] args) {
+ if (args.length == 3) {
+ host = args[0];
+ port = Integer.parseInt(args[1]);
+ topic = args[2];
+ return true;
+ } else {
+ System.err.println("Usage: KafkaProducerExample <host> <port> <topic>");
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java
new file mode 100644
index 0000000..4286196
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.connectors.kafka;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.api.java.typeutils.runtime.ByteArrayInputView;
+import org.apache.flink.runtime.util.DataOutputSerializer;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.flink.streaming.util.serialization.SerializationSchema;
+
+import java.io.IOException;
+
+public class Utils {
+ public static class TypeInformationSerializationSchema<T>
+ implements DeserializationSchema<T>, SerializationSchema<T, byte[]> {
+ private final TypeSerializer<T> serializer;
+ private final TypeInformation<T> ti;
+
+ public TypeInformationSerializationSchema(Object type, ExecutionConfig ec) {
+ this.ti = (TypeInformation<T>) TypeExtractor.getForObject(type);
+ this.serializer = ti.createSerializer(ec);
+ }
+ @Override
+ public T deserialize(byte[] message) {
+ try {
+ return serializer.deserialize(new ByteArrayInputView(message));
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to deserialize message", e);
+ }
+ }
+
+ @Override
+ public boolean isEndOfStream(T nextElement) {
+ return false;
+ }
+
+ @Override
+ public byte[] serialize(T element) {
+ DataOutputSerializer dos = new DataOutputSerializer(16);
+ try {
+ serializer.serialize(element, dos);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to serialize record", e);
+ }
+ return dos.getByteArray();
+ }
+
+ @Override
+ public TypeInformation<T> getProducedType() {
+ return ti;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java
new file mode 100644
index 0000000..0965b29
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka.api;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.flink.api.java.ClosureCleaner;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.flink.streaming.connectors.kafka.api.config.PartitionerWrapper;
+import org.apache.flink.streaming.connectors.kafka.partitioner.SerializableKafkaPartitioner;
+import org.apache.flink.streaming.util.serialization.SerializationSchema;
+import org.apache.flink.util.NetUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+import kafka.javaapi.producer.Producer;
+import kafka.producer.KeyedMessage;
+import kafka.producer.ProducerConfig;
+import kafka.serializer.DefaultEncoder;
+
+
+/**
+ * Sink that emits its inputs to a Kafka topic.
+ *
+ * @param <IN>
+ * Type of the sink input
+ */
+public class KafkaSink<IN> extends RichSinkFunction<IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSink.class);
+
+ private Producer<IN, byte[]> producer;
+ private Properties userDefinedProperties;
+ private String topicId;
+ private String brokerList;
+ private SerializationSchema<IN, byte[]> schema;
+ private SerializableKafkaPartitioner partitioner;
+ private Class<? extends SerializableKafkaPartitioner> partitionerClass = null;
+
+ /**
+ * Creates a KafkaSink for a given topic. The sink produces its input to
+ * the topic.
+ *
+ * @param brokerList
+ * Addresses of the brokers
+ * @param topicId
+ * ID of the Kafka topic.
+ * @param serializationSchema
+ * User defined serialization schema.
+ */
+ public KafkaSink(String brokerList, String topicId,
+ SerializationSchema<IN, byte[]> serializationSchema) {
+ this(brokerList, topicId, new Properties(), serializationSchema);
+ }
+
+ /**
+ * Creates a KafkaSink for a given topic with custom Producer configuration.
+ * If you use this constructor, the broker should be set with the "metadata.broker.list"
+ * configuration.
+ *
+ * @param brokerList
+ * Addresses of the brokers
+ * @param topicId
+ * ID of the Kafka topic.
+ * @param producerConfig
+ * Configurations of the Kafka producer
+ * @param serializationSchema
+ * User defined serialization schema.
+ */
+ public KafkaSink(String brokerList, String topicId, Properties producerConfig,
+ SerializationSchema<IN, byte[]> serializationSchema) {
+ String[] elements = brokerList.split(",");
+ for(String broker: elements) {
+ NetUtils.ensureCorrectHostnamePort(broker);
+ }
+ Preconditions.checkNotNull(topicId, "TopicID not set");
+
+ this.brokerList = brokerList;
+ this.topicId = topicId;
+ this.schema = serializationSchema;
+ this.partitionerClass = null;
+ this.userDefinedProperties = producerConfig;
+ }
+
+ /**
+ * Creates a KafkaSink for a given topic. The sink produces its input to
+ * the topic.
+ *
+ * @param brokerList
+ * @param topicId
+ * ID of the Kafka topic.
+ * @param serializationSchema
+ * User defined serialization schema.
+ * @param partitioner
+ * User defined partitioner.
+ */
+ public KafkaSink(String brokerList, String topicId,
+ SerializationSchema<IN, byte[]> serializationSchema, SerializableKafkaPartitioner partitioner) {
+ this(brokerList, topicId, serializationSchema);
+ ClosureCleaner.ensureSerializable(partitioner);
+ this.partitioner = partitioner;
+ }
+
+ public KafkaSink(String brokerList,
+ String topicId,
+ SerializationSchema<IN, byte[]> serializationSchema,
+ Class<? extends SerializableKafkaPartitioner> partitioner) {
+ this(brokerList, topicId, serializationSchema);
+ this.partitionerClass = partitioner;
+ }
+
+ /**
+ * Initializes the connection to Kafka.
+ */
+ @Override
+ public void open(Configuration configuration) {
+
+ Properties properties = new Properties();
+
+ properties.put("metadata.broker.list", brokerList);
+ properties.put("request.required.acks", "-1");
+ properties.put("message.send.max.retries", "10");
+
+ properties.put("serializer.class", DefaultEncoder.class.getCanonicalName());
+
+ // this will not be used as the key will not be serialized
+ properties.put("key.serializer.class", DefaultEncoder.class.getCanonicalName());
+
+ for (Map.Entry<Object, Object> propertiesEntry : userDefinedProperties.entrySet()) {
+ properties.put(propertiesEntry.getKey(), propertiesEntry.getValue());
+ }
+
+ if (partitioner != null) {
+ properties.put("partitioner.class", PartitionerWrapper.class.getCanonicalName());
+ // java serialization will do the rest.
+ properties.put(PartitionerWrapper.SERIALIZED_WRAPPER_NAME, partitioner);
+ }
+ if (partitionerClass != null) {
+ properties.put("partitioner.class", partitionerClass);
+ }
+
+ ProducerConfig config = new ProducerConfig(properties);
+
+ try {
+ producer = new Producer<IN, byte[]>(config);
+ } catch (NullPointerException e) {
+ throw new RuntimeException("Cannot connect to Kafka broker " + brokerList, e);
+ }
+ }
+
+ /**
+ * Called when new data arrives to the sink, and forwards it to Kafka.
+ *
+ * @param next
+ * The incoming data
+ */
+ @Override
+ public void invoke(IN next) {
+ byte[] serialized = schema.serialize(next);
+
+ // Sending message without serializable key.
+ producer.send(new KeyedMessage<IN, byte[]>(topicId, null, next, serialized));
+ }
+
+ @Override
+ public void close() {
+ if (producer != null) {
+ producer.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java
new file mode 100644
index 0000000..4a7ec15
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka.api;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import com.google.common.base.Preconditions;
+import kafka.consumer.Consumer;
+import kafka.consumer.ConsumerConfig;
+import kafka.consumer.ConsumerIterator;
+import kafka.consumer.KafkaStream;
+import kafka.javaapi.consumer.ConsumerConnector;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.source.ConnectorSource;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Source that listens to a Kafka topic using the high level Kafka API.
+ *
+ * @param <OUT>
+ * Type of the messages on the topic.
+ */
+public class KafkaSource<OUT> extends ConnectorSource<OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSource.class);
+
+ private final String zookeeperAddress;
+ private final String groupId;
+ private final String topicId;
+ private Properties customProperties;
+
+ private transient ConsumerConnector consumer;
+ private transient ConsumerIterator<byte[], byte[]> consumerIterator;
+
+ private long zookeeperSyncTimeMillis;
+ private static final long ZOOKEEPER_DEFAULT_SYNC_TIME = 200;
+ private static final String DEFAULT_GROUP_ID = "flink-group";
+
+ // We must read this in reachedEnd() to check for the end. We keep it to return it in
+ // next()
+ private OUT nextElement;
+
+ /**
+ * Creates a KafkaSource that consumes a topic.
+ *
+ * @param zookeeperAddress
+ * Address of the Zookeeper host (with port number).
+ * @param topicId
+ * ID of the Kafka topic.
+ * @param groupId
+ * ID of the consumer group.
+ * @param deserializationSchema
+ * User defined deserialization schema.
+ * @param zookeeperSyncTimeMillis
+ * Synchronization time with zookeeper.
+ */
+ public KafkaSource(String zookeeperAddress,
+ String topicId, String groupId,
+ DeserializationSchema<OUT> deserializationSchema,
+ long zookeeperSyncTimeMillis) {
+ this(zookeeperAddress, topicId, groupId, deserializationSchema, zookeeperSyncTimeMillis, null);
+ }
+ /**
+ * Creates a KafkaSource that consumes a topic.
+ *
+ * @param zookeeperAddress
+ * Address of the Zookeeper host (with port number).
+ * @param topicId
+ * ID of the Kafka topic.
+ * @param groupId
+ * ID of the consumer group.
+ * @param deserializationSchema
+ * User defined deserialization schema.
+ * @param zookeeperSyncTimeMillis
+ * Synchronization time with zookeeper.
+ * @param customProperties
+ * Custom properties for Kafka
+ */
+ public KafkaSource(String zookeeperAddress,
+ String topicId, String groupId,
+ DeserializationSchema<OUT> deserializationSchema,
+ long zookeeperSyncTimeMillis, Properties customProperties) {
+ super(deserializationSchema);
+ Preconditions.checkNotNull(zookeeperAddress, "ZK address is null");
+ Preconditions.checkNotNull(topicId, "Topic ID is null");
+ Preconditions.checkNotNull(deserializationSchema, "deserializationSchema is null");
+ Preconditions.checkArgument(zookeeperSyncTimeMillis >= 0, "The ZK sync time must be positive");
+
+ this.zookeeperAddress = zookeeperAddress;
+ this.groupId = groupId;
+ this.topicId = topicId;
+ this.zookeeperSyncTimeMillis = zookeeperSyncTimeMillis;
+ this.customProperties = customProperties;
+ }
+
+ /**
+ * Creates a KafkaSource that consumes a topic.
+ *
+ * @param zookeeperAddress
+ * Address of the Zookeeper host (with port number).
+ * @param topicId
+ * ID of the Kafka topic.
+ * @param deserializationSchema
+ * User defined deserialization schema.
+ * @param zookeeperSyncTimeMillis
+ * Synchronization time with zookeeper.
+ */
+ public KafkaSource(String zookeeperAddress, String topicId, DeserializationSchema<OUT> deserializationSchema, long zookeeperSyncTimeMillis) {
+ this(zookeeperAddress, topicId, DEFAULT_GROUP_ID, deserializationSchema, zookeeperSyncTimeMillis, null);
+ }
+ /**
+ * Creates a KafkaSource that consumes a topic.
+ *
+ * @param zookeeperAddress
+ * Address of the Zookeeper host (with port number).
+ * @param topicId
+ * ID of the Kafka topic.
+ * @param deserializationSchema
+ * User defined deserialization schema.
+ */
+ public KafkaSource(String zookeeperAddress, String topicId, DeserializationSchema<OUT> deserializationSchema) {
+ this(zookeeperAddress, topicId, deserializationSchema, ZOOKEEPER_DEFAULT_SYNC_TIME);
+ }
+
+ /**
+ * Initializes the connection to Kafka.
+ */
+ private void initializeConnection() {
+ Properties props = new Properties();
+ props.put("zookeeper.connect", zookeeperAddress);
+ props.put("group.id", groupId);
+ props.put("zookeeper.session.timeout.ms", "10000");
+ props.put("zookeeper.sync.time.ms", Long.toString(zookeeperSyncTimeMillis));
+ props.put("auto.commit.interval.ms", "1000");
+
+ if(customProperties != null) {
+ for(Map.Entry<Object, Object> e : props.entrySet()) {
+ if(props.contains(e.getKey())) {
+ LOG.warn("Overwriting property "+e.getKey()+" with value "+e.getValue());
+ }
+ props.put(e.getKey(), e.getValue());
+ }
+ }
+
+ consumer = Consumer.createJavaConsumerConnector(new ConsumerConfig(props));
+
+ Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer
+ .createMessageStreams(Collections.singletonMap(topicId, 1));
+ List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topicId);
+ KafkaStream<byte[], byte[]> stream = streams.get(0);
+
+ consumer.commitOffsets();
+
+ consumerIterator = stream.iterator();
+ }
+
+ @Override
+ public void open(Configuration config) throws Exception {
+ super.open(config);
+ initializeConnection();
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ if (consumer != null) {
+ consumer.shutdown();
+ }
+ }
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ if (nextElement != null) {
+ return false;
+ } else if (consumerIterator.hasNext()) {
+ OUT out = schema.deserialize(consumerIterator.next().message());
+ if (schema.isEndOfStream(out)) {
+ return true;
+ }
+ nextElement = out;
+ }
+ return false;
+ }
+
+ @Override
+ public OUT next() throws Exception {
+ if (!reachedEnd()) {
+ OUT result = nextElement;
+ nextElement = null;
+ return result;
+ } else {
+ throw new RuntimeException("Source exhausted");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/config/PartitionerWrapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/config/PartitionerWrapper.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/config/PartitionerWrapper.java
new file mode 100644
index 0000000..7ae17df
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/config/PartitionerWrapper.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.connectors.kafka.api.config;
+
+import kafka.producer.Partitioner;
+import kafka.utils.VerifiableProperties;
+
+/**
+ * Hacky wrapper to send an object instance through a Properties - map.
+ *
+ * This works as follows:
+ * The recommended way of creating a KafkaSink is specifying a classname for the partitioner.
+ *
+ * Otherwise (if the user gave a (serializable) class instance), we give Kafka the PartitionerWrapper class of Flink.
+ * This is set in the key-value (java.util.Properties) map.
+ * In addition to that, we use the Properties.put(Object, Object) to store the instance of the (serializable).
+ * This is a hack because the put() method is called on the underlying Hashmap.
+ *
+ * This PartitionerWrapper is called with the Properties. From there, we extract the wrapped Partitioner instance.
+ *
+ * The serializable PartitionerWrapper is serialized into the Properties Hashmap and also deserialized from there.
+ */
+public class PartitionerWrapper implements Partitioner {
+ public final static String SERIALIZED_WRAPPER_NAME = "flink.kafka.wrapper.serialized";
+
+ private Partitioner wrapped;
+ public PartitionerWrapper(VerifiableProperties properties) {
+ wrapped = (Partitioner) properties.props().get(SERIALIZED_WRAPPER_NAME);
+ }
+
+ @Override
+ public int partition(Object value, int numberOfPartitions) {
+ return wrapped.partition(value, numberOfPartitions);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java
new file mode 100644
index 0000000..032ed08
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java
@@ -0,0 +1,338 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka.api.persistent;
+
+import com.google.common.base.Preconditions;
+import kafka.common.TopicAndPartition;
+import kafka.consumer.Consumer;
+import kafka.consumer.ConsumerConfig;
+import kafka.consumer.ConsumerIterator;
+import kafka.consumer.KafkaStream;
+import kafka.javaapi.consumer.ConsumerConnector;
+import kafka.message.MessageAndMetadata;
+import kafka.utils.ZKGroupTopicDirs;
+import kafka.utils.ZkUtils;
+import org.I0Itec.zkclient.ZkClient;
+import org.I0Itec.zkclient.exception.ZkMarshallingError;
+import org.I0Itec.zkclient.serialize.ZkSerializer;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.checkpoint.CheckpointCommitter;
+import org.apache.flink.streaming.api.checkpoint.CheckpointedAsynchronously;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Option;
+import scala.collection.JavaConversions;
+import scala.collection.Seq;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Source for reading from Kafka using Flink Streaming Fault Tolerance.
+ * This source is updating the committed offset in Zookeeper based on the internal checkpointing of Flink.
+ *
+ * Note that the autocommit feature of Kafka needs to be disabled for using this source.
+ */
+public class PersistentKafkaSource<OUT> extends RichParallelSourceFunction<OUT> implements
+ ResultTypeQueryable<OUT>,
+ CheckpointCommitter,
+ CheckpointedAsynchronously<long[]> {
+ private static final Logger LOG = LoggerFactory.getLogger(PersistentKafkaSource.class);
+
+ protected transient ConsumerConfig consumerConfig;
+ private transient ConsumerIterator<byte[], byte[]> iteratorToRead;
+ private transient ConsumerConnector consumer;
+
+ private String topicName;
+ private DeserializationSchema<OUT> deserializationSchema;
+ private boolean running = true;
+
+ private transient long[] lastOffsets;
+ private transient ZkClient zkClient;
+ private transient long[] commitedOffsets; // maintain committed offsets, to avoid committing the same over and over again.
+
+ // We set this in reachedEnd to carry it over to next()
+ private OUT nextElement = null;
+
+ /**
+ *
+ * For the @param consumerConfig, specify at least the "groupid" and "zookeeper.connect" string.
+ * The config will be passed into the Kafka High Level Consumer.
+ * For a full list of possible values, check this out: https://kafka.apache.org/documentation.html#consumerconfigs
+ */
+ public PersistentKafkaSource(String topicName, DeserializationSchema<OUT> deserializationSchema, ConsumerConfig consumerConfig) {
+ Preconditions.checkNotNull(topicName);
+ Preconditions.checkNotNull(deserializationSchema);
+ Preconditions.checkNotNull(consumerConfig);
+
+ this.topicName = topicName;
+ this.deserializationSchema = deserializationSchema;
+ this.consumerConfig = consumerConfig;
+ if(consumerConfig.autoCommitEnable()) {
+ throw new IllegalArgumentException("'auto.commit.enable' is set to 'true'. " +
+ "This source can only be used with auto commit disabled because the " +
+ "source is committing to zookeeper by itself (not using the KafkaConsumer).");
+ }
+ if(!consumerConfig.offsetsStorage().equals("zookeeper")) {
+ // we can currently only commit to ZK.
+ throw new IllegalArgumentException("The 'offsets.storage' has to be set to 'zookeeper' for this Source to work reliably");
+ }
+ }
+
+ // ---------------------- ParallelSourceFunction Lifecycle -----------------
+
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ ConsumerConnector consumer = Consumer.createJavaConsumerConnector(this.consumerConfig);
+ // we request only one stream per consumer instance. Kafka will make sure that each consumer group
+ // will see each message only once.
+ Map<String,Integer> topicCountMap = Collections.singletonMap(topicName, 1);
+ Map<String, List<KafkaStream<byte[], byte[]>>> streams = consumer.createMessageStreams(topicCountMap);
+ if(streams.size() != 1) {
+ throw new RuntimeException("Expected only one message stream but got "+streams.size());
+ }
+ List<KafkaStream<byte[], byte[]>> kafkaStreams = streams.get(topicName);
+ if(kafkaStreams == null) {
+ throw new RuntimeException("Requested stream not available. Available streams: "+streams.toString());
+ }
+ if(kafkaStreams.size() != 1) {
+ throw new RuntimeException("Requested 1 stream from Kafka, bot got "+kafkaStreams.size()+" streams");
+ }
+ LOG.info("Opening Consumer instance for topic '{}' on group '{}'", topicName, consumerConfig.groupId());
+ this.iteratorToRead = kafkaStreams.get(0).iterator();
+ this.consumer = consumer;
+
+ zkClient = new ZkClient(consumerConfig.zkConnect(),
+ consumerConfig.zkSessionTimeoutMs(),
+ consumerConfig.zkConnectionTimeoutMs(),
+ new KafkaZKStringSerializer());
+
+ // most likely the number of offsets we're going to store here will be lower than the number of partitions.
+ int numPartitions = getNumberOfPartitions();
+ LOG.debug("The topic {} has {} partitions", topicName, numPartitions);
+ this.lastOffsets = new long[numPartitions];
+ this.commitedOffsets = new long[numPartitions];
+ Arrays.fill(this.lastOffsets, -1);
+ Arrays.fill(this.commitedOffsets, 0); // just to make it clear
+
+ nextElement = null;
+ }
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ if (nextElement != null) {
+ return false;
+ }
+
+ while (iteratorToRead.hasNext()) {
+ MessageAndMetadata<byte[], byte[]> message = iteratorToRead.next();
+ if(lastOffsets[message.partition()] >= message.offset()) {
+ LOG.info("Skipping message with offset {} from partition {}", message.offset(), message.partition());
+ continue;
+ }
+ lastOffsets[message.partition()] = message.offset();
+
+ OUT out = deserializationSchema.deserialize(message.message());
+ if (deserializationSchema.isEndOfStream(out)) {
+ LOG.info("DeserializationSchema signaled end of stream for this source");
+ break;
+ }
+
+ nextElement = out;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Processed record with offset {} from partition {}", message.offset(), message.partition());
+ }
+ break;
+ }
+
+ return nextElement == null;
+ }
+
+ @Override
+ public OUT next() throws Exception {
+ if (!reachedEnd()) {
+ OUT result = nextElement;
+ nextElement = null;
+ return result;
+ } else {
+ throw new RuntimeException("Source exhausted");
+ }
+ }
+
+ @Override
+ public void close() {
+ LOG.info("Closing Kafka consumer");
+ this.consumer.shutdown();
+ zkClient.close();
+ }
+
+
+ // ---------------------- State / Checkpoint handling -----------------
+ // this source is keeping the partition offsets in Zookeeper
+
+ private Map<Long, long[]> pendingCheckpoints = new HashMap<Long, long[]>();
+
+ @Override
+ public long[] snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
+ if(lastOffsets == null) {
+ LOG.warn("State snapshot requested on not yet opened source. Returning null");
+ return null;
+ }
+ LOG.info("Snapshotting state. Offsets: {}, checkpoint id {}, timestamp {}", Arrays.toString(lastOffsets), checkpointId, checkpointTimestamp);
+
+ long[] currentOffsets = Arrays.copyOf(lastOffsets, lastOffsets.length);
+ pendingCheckpoints.put(checkpointId, currentOffsets);
+ return currentOffsets;
+ }
+
+ @Override
+ public void restoreState(long[] state) {
+ // we maintain the offsets in Kafka, so nothing to do.
+ }
+
+
+ /**
+ * Notification on completed checkpoints
+ * @param checkpointId The ID of the checkpoint that has been completed.
+ */
+ @Override
+ public void commitCheckpoint(long checkpointId) {
+ LOG.info("Commit checkpoint {}", checkpointId);
+ long[] checkpointOffsets = pendingCheckpoints.remove(checkpointId);
+ if(checkpointOffsets == null) {
+ LOG.warn("Unable to find pending checkpoint for id {}", checkpointId);
+ return;
+ }
+ LOG.info("Got corresponding offsets {}", Arrays.toString(checkpointOffsets));
+
+ for(int partition = 0; partition < checkpointOffsets.length; partition++) {
+ long offset = checkpointOffsets[partition];
+ if(offset != -1) {
+ setOffset(partition, offset);
+ }
+ }
+ }
+
+ // --------------------- Zookeeper / Offset handling -----------------------------
+
+ private int getNumberOfPartitions() {
+ scala.collection.immutable.List<String> scalaSeq = JavaConversions.asScalaBuffer(Collections.singletonList(topicName)).toList();
+ scala.collection.mutable.Map<String, Seq<Object>> list = ZkUtils.getPartitionsForTopics(zkClient, scalaSeq);
+ Option<Seq<Object>> topicOption = list.get(topicName);
+ if(topicOption.isEmpty()) {
+ throw new IllegalArgumentException("Unable to get number of partitions for topic "+topicName+" from "+list.toString());
+ }
+ Seq<Object> topic = topicOption.get();
+ return topic.size();
+ }
+
+ protected void setOffset(int partition, long offset) {
+ if(commitedOffsets[partition] < offset) {
+ setOffset(zkClient, consumerConfig.groupId(), topicName, partition, offset);
+ commitedOffsets[partition] = offset;
+ } else {
+ LOG.debug("Ignoring offset {} for partition {} because it is already committed", offset, partition);
+ }
+ }
+
+
+
+ // the following two methods are static to allow access from the outside as well (Testcases)
+
+ /**
+ * This method's code is based on ZookeeperConsumerConnector.commitOffsetToZooKeeper()
+ */
+ public static void setOffset(ZkClient zkClient, String groupId, String topic, int partition, long offset) {
+ LOG.info("Setting offset for partition {} of topic {} in group {} to offset {}", partition, topic, groupId, offset);
+ TopicAndPartition tap = new TopicAndPartition(topic, partition);
+ ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, tap.topic());
+ ZkUtils.updatePersistentPath(zkClient, topicDirs.consumerOffsetDir() + "/" + tap.partition(), Long.toString(offset));
+ }
+
+ public static long getOffset(ZkClient zkClient, String groupId, String topic, int partition) {
+ TopicAndPartition tap = new TopicAndPartition(topic, partition);
+ ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, tap.topic());
+ scala.Tuple2<String, Stat> data = ZkUtils.readData(zkClient, topicDirs.consumerOffsetDir() + "/" + tap.partition());
+ return Long.valueOf(data._1());
+ }
+
+
+ // ---------------------- (Java)Serialization methods for the consumerConfig -----------------
+
+ private void writeObject(ObjectOutputStream out)
+ throws IOException, ClassNotFoundException {
+ out.defaultWriteObject();
+ out.writeObject(consumerConfig.props().props());
+ }
+
+ private void readObject(ObjectInputStream in)
+ throws IOException, ClassNotFoundException {
+ in.defaultReadObject();
+ Properties props = (Properties) in.readObject();
+ consumerConfig = new ConsumerConfig(props);
+ }
+
+
+ @Override
+ public TypeInformation<OUT> getProducedType() {
+ return deserializationSchema.getProducedType();
+ }
+
+
+ // ---------------------- Zookeeper Serializer copied from Kafka (because it has private access there) -----------------
+
+ public static class KafkaZKStringSerializer implements ZkSerializer {
+
+ @Override
+ public byte[] serialize(Object data) throws ZkMarshallingError {
+ try {
+ return ((String) data).getBytes("UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Object deserialize(byte[] bytes) throws ZkMarshallingError {
+ if (bytes == null) {
+ return null;
+ } else {
+ try {
+ return new String(bytes, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/KafkaConstantPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/KafkaConstantPartitioner.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/KafkaConstantPartitioner.java
new file mode 100644
index 0000000..661d0bd
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/KafkaConstantPartitioner.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka.partitioner;
+
+public class KafkaConstantPartitioner implements SerializableKafkaPartitioner {
+
+ private static final long serialVersionUID = 1L;
+ private int partition;
+
+ public KafkaConstantPartitioner(int partition) {
+ this.partition = partition;
+ }
+
+ @Override
+ public int partition(Object value, int numberOfPartitions) {
+ return partition;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/SerializableKafkaPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/SerializableKafkaPartitioner.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/SerializableKafkaPartitioner.java
new file mode 100644
index 0000000..77a774e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/partitioner/SerializableKafkaPartitioner.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.connectors.kafka.partitioner;
+
+import kafka.producer.Partitioner;
+import java.io.Serializable;
+
+public interface SerializableKafkaPartitioner extends Serializable, Partitioner {
+
+}
[7/7] flink git commit: [FLINK-2088] [streaming] [scala]
DataStream.name() returns a typed DataStream
Posted by mb...@apache.org.
[FLINK-2088] [streaming] [scala] DataStream.name() returns a typed DataStream
Closes #739
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7f5edeb9
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7f5edeb9
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7f5edeb9
Branch: refs/heads/master
Commit: 7f5edeb9a7fa7e8a279634437be36cf3594a67b1
Parents: a302817
Author: mbalassi <mb...@apache.org>
Authored: Thu May 28 09:58:40 2015 +0200
Committer: mbalassi <mb...@apache.org>
Committed: Thu May 28 14:45:47 2015 +0200
----------------------------------------------------------------------
.../scala/org/apache/flink/streaming/api/scala/DataStream.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/7f5edeb9/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
index 89fd999..3618c49 100644
--- a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
@@ -93,7 +93,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
* @return The named operator
*/
- def name(name: String) : DataStream[_] = javaStream match {
+ def name(name: String) : DataStream[T] = javaStream match {
case stream : SingleOutputStreamOperator[_,_] => javaStream.name(name)
case _ => throw new
UnsupportedOperationException("Only supported for operators.")
[6/7] flink git commit: [docs] [streaming] Flume connector
temporarily removed from docs
Posted by mb...@apache.org.
[docs] [streaming] Flume connector temporarily removed from docs
As the Flume source is not working currently.
Closes #719
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a3028173
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a3028173
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a3028173
Branch: refs/heads/master
Commit: a30281738529c36c4a8f0ade6ab50079b8d69be4
Parents: 665bcec
Author: mbalassi <mb...@apache.org>
Authored: Sat May 23 10:52:07 2015 +0200
Committer: mbalassi <mb...@apache.org>
Committed: Thu May 28 14:45:18 2015 +0200
----------------------------------------------------------------------
docs/apis/streaming_guide.md | 156 +-------------------------------------
1 file changed, 4 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a3028173/docs/apis/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/apis/streaming_guide.md b/docs/apis/streaming_guide.md
index 6565fbf..543499c 100644
--- a/docs/apis/streaming_guide.md
+++ b/docs/apis/streaming_guide.md
@@ -23,7 +23,7 @@ under the License.
<a href="#top"></a>
-Flink Streaming is a system for high-throughput, low-latency data stream processing. The system can connect to and process data streams from different data sources like file sources, web sockets, message queues(Apache Kafka, RabbitMQ, Apache Flume, Twitter…) and also from any user defined data source using a very simple interface. Data streams can be transformed and modified to create new data streams using high-level functions similar to the ones provided by the batch processing API. Flink Streaming natively supports flexible, data-driven windowing semantics and iterative stream processing. The processed data can be pushed to different output types.
+Flink Streaming is a system for high-throughput, low-latency data stream processing. The system can connect to and process data streams from different data sources like file sources, web sockets, message queues(Apache Kafka, RabbitMQ, Twitter Streaming API …) and also from any user defined data source using a very simple interface. Data streams can be transformed and modified to create new data streams using high-level functions similar to the ones provided by the batch processing API. Flink Streaming natively supports flexible, data-driven windowing semantics and iterative stream processing. The processed data can be pushed to different output types.
* This will be replaced by the TOC
{:toc}
@@ -1205,7 +1205,8 @@ To minimise latency, set the timeout to a value close to 0 (for example 5 or 10
Stream connectors
----------------
-Connectors provide an interface for accessing data from various third party sources (message queues). Currently four connectors are natively supported, namely [Apache Kafka](https://kafka.apache.org/), [RabbitMQ](http://www.rabbitmq.com/), [Apache Flume](https://flume.apache.org/index.html) and [Twitter Streaming API](https://dev.twitter.com/docs/streaming-apis).
+<!-- TODO: reintroduce flume -->
+Connectors provide an interface for accessing data from various third party sources (message queues). Currently three connectors are natively supported, namely [Apache Kafka](https://kafka.apache.org/), [RabbitMQ](http://www.rabbitmq.com/) and the [Twitter Streaming API](https://dev.twitter.com/docs/streaming-apis).
Typically the connector packages consist of a source and sink class (with the exception of Twitter where only a source is provided). To use these sources the user needs to pass Serialization/Deserialization schemas for the connectors for the desired types. (Or use some predefined ones)
@@ -1347,98 +1348,6 @@ More about Kafka can be found [here](https://kafka.apache.org/documentation.html
[Back to top](#top)
-### Apache Flume
-
-This connector provides access to data streams from [Apache Flume](http://flume.apache.org/).
-
-#### Installing Apache Flume
-[Download](http://flume.apache.org/download.html) Apache Flume. A configuration file is required for starting agents in Flume. A configuration file for running the example can be found [here](#config_file).
-
-#### Flume Source
-A class providing an interface for receiving data from Flume.
-
-The followings have to be provided for the `FlumeSource(…)` constructor in order:
-
-1. The hostname
-2. The port number
-3. Deserialization schema
-
-Example:
-
-<div class="codetabs" markdown="1">
-<div data-lang="java" markdown="1">
-{% highlight java %}
-DataStream<String> stream = env
- .addSource(new FlumeSource<String>("localhost", 41414, new SimpleStringSchema()))
- .print();
-{% endhighlight %}
-</div>
-<div data-lang="scala" markdown="1">
-{% highlight scala %}
-stream = env
- .addSource(new FlumeSource[String]("localhost", 41414, new SimpleStringSchema))
- .print
-{% endhighlight %}
-</div>
-</div>
-
-#### Flume Sink
-A class providing an interface for sending data to Flume.
-
-The followings have to be provided for the `FlumeSink(…)` constructor in order:
-
-1. The hostname
-2. The port number
-3. Serialisation schema
-
-Example:
-
-<div class="codetabs" markdown="1">
-<div data-lang="java" markdown="1">
-{% highlight java %}
-stream.addSink(new FlumeSink<String>("localhost", 42424, new StringToByteSerializer()));
-{% endhighlight %}
-</div>
-<div data-lang="scala" markdown="1">
-{% highlight scala %}
-stream.addSink(new FlumeSink[String]("localhost", 42424, new StringToByteSerializer))
-{% endhighlight %}
-</div>
-</div>
-
-##### Configuration file<a name="config_file"></a>
-An example of a configuration file:
-
-~~~
-a1.channels = c1
-a1.sources = r1
-a1.sinks = k1
-
-a1.channels.c1.type = memory
-
-a1.sources.r1.channels = c1
-a1.sources.r1.type = avro
-a1.sources.r1.bind = localhost
-a1.sources.r1.port = 42424
-
-a1.sinks.k1.channel = c1
-a1.sinks.k1.type = avro
-a1.sinks.k1.hostname = localhost
-a1.sinks.k1.port = 41414
-~~~
-
-To run the `FlumeTopology` example the previous configuration file must located in the Flume directory and named example.conf and the agent can be started with the following command:
-
-~~~
-bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger=INFO,console
-~~~
-
-If the agent is not started before the application starts a `FlumeSink` then the sink will retry to build the connection for 90 seconds, if unsuccessful it throws a `RuntimeException`.
-
-More on Flume can be found [here](http://flume.apache.org).
-
-[Back to top](#top)
-
### RabbitMQ
This connector provides access to data streams from [RabbitMQ](http://www.rabbitmq.com/).For using this connector add the following dependency to your project:
@@ -1595,7 +1504,7 @@ There are two basic types of tweets. The usual tweets contain information such a
### Docker containers for connectors
-A Docker container is provided with all the required configurations for test running the connectors of Apache Flink. The servers for the message queues will be running on the docker container while the example topology can be run on the user's computer. The only exception is Flume, more can be read about this issue in the [Flume section](#flume).
+A Docker container is provided with all the required configurations for test running the connectors of Apache Flink. The servers for the message queues will be running on the docker container while the example topology can be run on the user's computer.
#### Installing Docker
The official Docker installation guide can be found [here](https://docs.docker.com/installation/).
@@ -1701,60 +1610,3 @@ In the example there are two connectors. One that sends messages to Kafka and on
<DATE> INFO kafka.KafkaTopology: String: (8) arrived from Kafka
<DATE> INFO kafka.KafkaTopology: String: (9) arrived from Kafka
~~~
-
-#### Apache Flume
-
-At the moment remote access for Flume connectors does not work. This example is only runnable on the same machine where the Flume server is. In this case both will be in the Docker container.
-
-Pull the image:
-
-~~~bash
-sudo docker pull flinkstreaming/flink-connectors-flume
-~~~
-
-To run the container type:
-
-~~~bash
-sudo docker run -t -i flinkstreaming/flink-connectors-flume
-~~~
-
-Now a terminal started running from the image with all the necessary configurations to test run the Flume connector. The -p flag binds the localhost's and the Docker container's ports so flume can communicate with the application through these.
-
-To have the latest version of Flink type:
-~~~bash
-cd /git/flink/
-git pull
-~~~
-
-Then build the code with:
-
-~~~bash
-cd /git/flink/flink-staging/flink-streaming/flink-streaming-connectors/
-mvn install -DskipTests
-~~~
-
-First start the server in the background:
-
-~~~bash
-/apache-flume-1.5.0-bin/bin/flume-ng agent \
---conf conf --conf-file /apache-flume-1.5.0-bin/example.conf --name a1 \
--Dflume.root.logger=INFO,console > /flumelog.txt 2> /flumeerr.txt &
-~~~
-
-Then press enter and launch the example with:
-
-~~~bash
-java -cp /PATH/TO/JAR-WITH-DEPENDENCIES org.apache.flink.streaming.connectors.flume.FlumeTopology
-~~~
-
-In the example there are to connectors. One that sends messages to Flume and one that receives messages from the same queue. In the logger messages the arriving messages can be observed in the following format:
-
-~~~
-<DATE> INFO flume.FlumeTopology: String: <one> arrived from Flume
-<DATE> INFO flume.FlumeTopology: String: <two> arrived from Flume
-<DATE> INFO flume.FlumeTopology: String: <three> arrived from Flume
-<DATE> INFO flume.FlumeTopology: String: <four> arrived from Flume
-<DATE> INFO flume.FlumeTopology: String: <five> arrived from Flume
-~~~
-
-[Back to top](#top)
[3/7] flink git commit: [FLINK-1874] [streaming] Connectors separated
into individual projects
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
new file mode 100644
index 0000000..4bc6df0
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.twitter;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
+import org.apache.flink.util.Collector;
+import org.apache.sling.commons.json.JSONException;
+
+/**
+ * This program demonstrate the use of TwitterSource.
+ * Its aim is to count the frequency of the languages of tweets
+ */
+public class TwitterTopology {
+
+ private static final int NUMBEROFTWEETS = 100;
+
+ /**
+ * FlatMapFunction to determine the language of tweets if possible
+ */
+ public static class SelectLanguageFlatMap extends
+ JSONParseFlatMap<String, String> {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Select the language from the incoming JSON text
+ */
+ @Override
+ public void flatMap(String value, Collector<String> out) throws Exception {
+ try{
+ out.collect(getString(value, "lang"));
+ }
+ catch (JSONException e){
+ out.collect("");
+ }
+ }
+
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ String path = new String();
+
+ if (args != null && args.length == 1) {
+ path = args[0];
+ } else {
+ System.err.println("USAGE:\nTwitterLocal <pathToPropertiesFile>");
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<String> streamSource = env.addSource(new TwitterSource(path, NUMBEROFTWEETS));
+
+
+ DataStream<Tuple2<String, Integer>> dataStream = streamSource
+ .flatMap(new SelectLanguageFlatMap())
+ .map(new MapFunction<String, Tuple2<String, Integer>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<String, Integer> map(String value) throws Exception {
+ return new Tuple2<String, Integer>(value, 1);
+ }
+ })
+ .groupBy(0)
+ .sum(1);
+
+ dataStream.print();
+
+ env.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
new file mode 100644
index 0000000..b1d4115
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.json;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.flink.streaming.connectors.json.JSONParser;
+import org.apache.sling.commons.json.JSONException;
+import org.apache.sling.commons.json.JSONObject;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class JSONParserTest {
+
+ private String jsonText;
+ private String searchedField;
+
+ public JSONParserTest(String text, String field) {
+ jsonText = text;
+ searchedField = field;
+ }
+
+ @Parameters
+ public static Collection<Object[]> initParameterList() {
+
+ Object[][] parameterList = new Object[][] {
+ { "{\"key\":\"value\"}", "key" },
+ { "{\"key\":[\"value\"]}", "key[0]" },
+ { "{\"key\":[{\"key\":\"value\"}]}", "key[0].key" },
+ { "{\"key\":[{\"key\":[{\"key\":\"value\"}]}]}", "key[0].key[0].key"},
+ { "{\"key\":[1,[{\"key\":\"value\"}]]}", "key[1][0].key" },
+ { "{\"key\":[1,[[\"key\",2,\"value\"]]]}", "key[1][0][2]" },
+ { "{\"key\":{\"key\":{\"otherKey\":\"wrongValue\",\"key\":\"value\"},\"otherKey\":\"wrongValue\"},\"otherKey\":\"wrongValue\"}" , "key.key.key"}
+ };
+
+ return Arrays.asList(parameterList);
+ }
+
+ @Test
+ public void test() {
+ try {
+ JSONParser parser = new JSONParser(jsonText);
+ JSONObject jo = parser.parse(searchedField);
+ String expected = "{\"retValue\":\"value\"}";
+
+ assertTrue(expected.equals(jo.toString()));
+ }
+ catch (JSONException e) {
+ fail();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
new file mode 100644
index 0000000..8851086
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/java/org/apache/flink/streaming/connectors/json/JSONParserTest2.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.json;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.flink.streaming.connectors.json.JSONParser;
+import org.apache.sling.commons.json.JSONException;
+import org.apache.sling.commons.json.JSONObject;
+import org.junit.Test;
+
+
+public class JSONParserTest2 {
+
+ @Test
+ public void testGetBooleanFunction() {
+ String jsonText = "{\"key\":true}";
+ String searchedField = "key";
+ try {
+ JSONParser parser = new JSONParser(jsonText);
+ JSONObject jo = parser.parse(searchedField);
+
+ assertTrue(jo.getBoolean("retValue"));
+ }
+ catch (JSONException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testGetDoubleFunction() {
+ double expected = 12345.12345;
+ String jsonText = "{\"key\":" + expected + "}";
+ String searchedField = "key";
+ try {
+ JSONParser parser = new JSONParser(jsonText);
+ JSONObject jo = parser.parse(searchedField);
+
+ assertEquals(expected,jo.getDouble("retValue"),0.000001);
+ }
+ catch (JSONException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testGetIntFunction() {
+ int expected = 15;
+ String jsonText = "{\"key\":" + expected + "}";
+ String searchedField = "key";
+ try {
+ JSONParser parser = new JSONParser(jsonText);
+ JSONObject jo = parser.parse(searchedField);
+
+ assertEquals(expected,jo.getInt("retValue"));
+ }
+ catch (JSONException e) {
+ fail();
+ }
+ }
+
+ @Test
+ public void testGetLongFunction() {
+ long expected = 111111111111L;
+ String jsonText = "{\"key\":" + expected + "}";
+ String searchedField = "key";
+ try {
+ JSONParser parser = new JSONParser(jsonText);
+ JSONObject jo = parser.parse(searchedField);
+
+ assertEquals(expected,jo.getLong("retValue"));
+ }
+ catch (JSONException e) {
+ fail();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/resources/log4j-test.properties b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..9ede613
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/resources/log4j-test.properties
@@ -0,0 +1,27 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=INFO, testlogger
+
+log4j.appender.testlogger=org.apache.log4j.ConsoleAppender
+log4j.appender.testlogger.target = System.err
+log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout
+log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+
+# suppress the irrelevant (wrong) warnings from the netty channel handler
+log4j.logger.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, testlogger
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/resources/logback-test.xml b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..45b3b92
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/test/resources/logback-test.xml
@@ -0,0 +1,30 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT"/>
+ </root>
+ <logger name="org.apache.flink.streaming" level="WARN"/>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml
index 1ecf6c1..4b82d09 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/pom.xml
@@ -30,215 +30,16 @@ under the License.
<relativePath>..</relativePath>
</parent>
- <artifactId>flink-streaming-connectors</artifactId>
+ <artifactId>flink-streaming-connectors-parent</artifactId>
<name>flink-streaming-connectors</name>
- <packaging>jar</packaging>
+ <packaging>pom</packaging>
- <!-- Allow users to pass custom connector versions -->
- <properties>
- <kafka.version>0.8.2.0</kafka.version>
- <rabbitmq.version>3.3.1</rabbitmq.version>
- <flume-ng.version>1.5.0</flume-ng.version>
- </properties>
+ <modules>
+ <module>flink-connector-flume</module>
+ <module>flink-connector-kafka</module>
+ <module>flink-connector-rabbitmq</module>
+ <module>flink-connector-twitter</module>
+ </modules>
- <dependencies>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_${scala.binary.version}</artifactId>
- <version>${kafka.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.sun.jmx</groupId>
- <artifactId>jmxri</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jdmk</groupId>
- <artifactId>jmxtools</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-simple</artifactId>
- </exclusion>
- <exclusion>
- <groupId>net.sf.jopt-simple</groupId>
- <artifactId>jopt-simple</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-reflect</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.yammer.metrics</groupId>
- <artifactId>metrics-annotation</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-test</artifactId>
- <version>2.7.1</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.rabbitmq</groupId>
- <artifactId>amqp-client</artifactId>
- <version>${rabbitmq.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flume</groupId>
- <artifactId>flume-ng-core</artifactId>
- <version>${flume-ng.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.tukaani</groupId>
- <artifactId>xz</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.velocity</groupId>
- <artifactId>velocity</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libthrift</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>com.twitter</groupId>
- <artifactId>hbc-core</artifactId>
- <version>2.2.0</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.sling</groupId>
- <artifactId>org.apache.sling.commons.json</artifactId>
- <version>2.0.6</version>
- </dependency>
-
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <!-- Override artifactSet configuration to build fat-jar with all dependencies packed. -->
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.3</version>
- <executions>
- <execution>
- <id>shade-flink</id>
- <configuration>
- <artifactSet>
- <includes combine.children="append">
- <!-- We include all dependencies that transitively depend on guava -->
- <include>org.apache.flume:*</include>
- <include>com.twitter:hbc-core</include>
- <include>com.twitter:joauth</include>
- </includes>
- </artifactSet>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
deleted file mode 100644
index fde4cdf..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
-import org.apache.flink.streaming.util.serialization.DeserializationSchema;
-
-public abstract class ConnectorSource<OUT> extends RichParallelSourceFunction<OUT> implements ResultTypeQueryable<OUT>{
-
- private static final long serialVersionUID = 1L;
- protected DeserializationSchema<OUT> schema;
-
- public ConnectorSource(DeserializationSchema<OUT> schema) {
- this.schema = schema;
- }
-
- @Override
- public TypeInformation<OUT> getProducedType() {
- return schema.getProducedType();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
deleted file mode 100644
index 50f5770..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.flume;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
-import org.apache.flink.streaming.util.serialization.SerializationSchema;
-import org.apache.flume.Event;
-import org.apache.flume.EventDeliveryException;
-import org.apache.flume.FlumeException;
-import org.apache.flume.api.RpcClient;
-import org.apache.flume.api.RpcClientFactory;
-import org.apache.flume.event.EventBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FlumeSink<IN> extends RichSinkFunction<IN> {
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(FlumeSink.class);
-
- private transient FlinkRpcClientFacade client;
- boolean initDone = false;
- String host;
- int port;
- SerializationSchema<IN, byte[]> schema;
-
- public FlumeSink(String host, int port, SerializationSchema<IN, byte[]> schema) {
- this.host = host;
- this.port = port;
- this.schema = schema;
- }
-
- /**
- * Receives tuples from the Apache Flink {@link DataStream} and forwards
- * them to Apache Flume.
- *
- * @param value
- * The tuple arriving from the datastream
- */
- @Override
- public void invoke(IN value) {
-
- byte[] data = schema.serialize(value);
- client.sendDataToFlume(data);
-
- }
-
- private class FlinkRpcClientFacade {
- private RpcClient client;
- private String hostname;
- private int port;
-
- /**
- * Initializes the connection to Apache Flume.
- *
- * @param hostname
- * The host
- * @param port
- * The port.
- */
- public void init(String hostname, int port) {
- // Setup the RPC connection
- this.hostname = hostname;
- this.port = port;
- int initCounter = 0;
- while (true) {
- if (initCounter >= 90) {
- throw new RuntimeException("Cannot establish connection with" + port + " at "
- + host);
- }
- try {
- this.client = RpcClientFactory.getDefaultInstance(hostname, port);
- } catch (FlumeException e) {
- // Wait one second if the connection failed before the next
- // try
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e1) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Interrupted while trying to connect {} at {}", port, host);
- }
- }
- }
- if (client != null) {
- break;
- }
- initCounter++;
- }
- initDone = true;
- }
-
- /**
- * Sends byte arrays as {@link Event} series to Apache Flume.
- *
- * @param data
- * The byte array to send to Apache FLume
- */
- public void sendDataToFlume(byte[] data) {
- Event event = EventBuilder.withBody(data);
-
- try {
- client.append(event);
-
- } catch (EventDeliveryException e) {
- // clean up and recreate the client
- client.close();
- client = null;
- client = RpcClientFactory.getDefaultInstance(hostname, port);
- }
- }
-
- }
-
- @Override
- public void close() {
- client.client.close();
- }
-
- @Override
- public void open(Configuration config) {
- client = new FlinkRpcClientFacade();
- client.init(host, port);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
deleted file mode 100644
index bb9ce38..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
+++ /dev/null
@@ -1,149 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one or more
-// * contributor license agreements. See the NOTICE file distributed with
-// * this work for additional information regarding copyright ownership.
-// * The ASF licenses this file to You under the Apache License, Version 2.0
-// * (the "License"); you may not use this file except in compliance with
-// * the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.flink.streaming.connectors.flume;
-//
-//import java.util.List;
-//
-//import org.apache.flink.streaming.api.datastream.DataStream;
-//import org.apache.flink.streaming.connectors.ConnectorSource;
-//import org.apache.flink.streaming.util.serialization.DeserializationSchema;
-//import org.apache.flink.util.Collector;
-//import org.apache.flume.Context;
-//import org.apache.flume.channel.ChannelProcessor;
-//import org.apache.flume.source.AvroSource;
-//import org.apache.flume.source.avro.AvroFlumeEvent;
-//import org.apache.flume.source.avro.Status;
-//
-//public class FlumeSource<OUT> extends ConnectorSource<OUT> {
-// private static final long serialVersionUID = 1L;
-//
-// String host;
-// String port;
-// volatile boolean finished = false;
-//
-// private volatile boolean isRunning = false;
-//
-// FlumeSource(String host, int port, DeserializationSchema<OUT> deserializationSchema) {
-// super(deserializationSchema);
-// this.host = host;
-// this.port = Integer.toString(port);
-// }
-//
-// public class MyAvroSource extends AvroSource {
-// Collector<OUT> output;
-//
-// /**
-// * Sends the AvroFlumeEvent from it's argument list to the Apache Flink
-// * {@link DataStream}.
-// *
-// * @param avroEvent
-// * The event that should be sent to the dataStream
-// * @return A {@link Status}.OK message if sending the event was
-// * successful.
-// */
-// @Override
-// public Status append(AvroFlumeEvent avroEvent) {
-// collect(avroEvent);
-// return Status.OK;
-// }
-//
-// /**
-// * Sends the AvroFlumeEvents from it's argument list to the Apache Flink
-// * {@link DataStream}.
-// *
-// * @param events
-// * The events that is sent to the dataStream
-// * @return A Status.OK message if sending the events was successful.
-// */
-// @Override
-// public Status appendBatch(List<AvroFlumeEvent> events) {
-// for (AvroFlumeEvent avroEvent : events) {
-// collect(avroEvent);
-// }
-//
-// return Status.OK;
-// }
-//
-// /**
-// * Deserializes the AvroFlumeEvent before sending it to the Apache Flink
-// * {@link DataStream}.
-// *
-// * @param avroEvent
-// * The event that is sent to the dataStream
-// */
-// private void collect(AvroFlumeEvent avroEvent) {
-// byte[] b = avroEvent.getBody().array();
-// OUT out = FlumeSource.this.schema.deserialize(b);
-//
-// if (schema.isEndOfStream(out)) {
-// FlumeSource.this.finished = true;
-// this.stop();
-// FlumeSource.this.notifyAll();
-// } else {
-// output.collect(out);
-// }
-//
-// }
-//
-// }
-//
-// MyAvroSource avroSource;
-//
-// /**
-// * Configures the AvroSource. Also sets the output so the application can
-// * use it from outside of the invoke function.
-// *
-// * @param output
-// * The output used in the invoke function
-// */
-// public void configureAvroSource(Collector<OUT> output) {
-//
-// avroSource = new MyAvroSource();
-// avroSource.output = output;
-// Context context = new Context();
-// context.put("port", port);
-// context.put("bind", host);
-// avroSource.configure(context);
-// // An instance of a ChannelProcessor is required for configuring the
-// // avroSource although it will not be used in this case.
-// ChannelProcessor cp = new ChannelProcessor(null);
-// avroSource.setChannelProcessor(cp);
-// }
-//
-// /**
-// * Configures the AvroSource and runs until the user calls a close function.
-// *
-// * @param output
-// * The Collector for sending data to the datastream
-// */
-// @Override
-// public void run(Collector<OUT> output) throws Exception {
-// isRunning = true;
-// configureAvroSource(output);
-// avroSource.start();
-// while (!finished && isRunning) {
-// this.wait();
-// }
-// }
-//
-// @Override
-// public void cancel() {
-// isRunning = false;
-// }
-//
-//}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
deleted file mode 100644
index f630bce..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
+++ /dev/null
@@ -1,49 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one or more
-// * contributor license agreements. See the NOTICE file distributed with
-// * this work for additional information regarding copyright ownership.
-// * The ASF licenses this file to You under the Apache License, Version 2.0
-// * (the "License"); you may not use this file except in compliance with
-// * the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.flink.streaming.connectors.flume;
-//
-//import org.apache.flink.streaming.api.datastream.DataStream;
-//import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-//import org.apache.flink.streaming.util.serialization.SerializationSchema;
-//import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
-//
-//public class FlumeTopology {
-//
-// public static void main(String[] args) throws Exception {
-//
-// StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-//
-// @SuppressWarnings("unused")
-// DataStream<String> dataStream1 = env.addSource(
-// new FlumeSource<String>("localhost", 41414, new SimpleStringSchema())).addSink(
-// new FlumeSink<String>("localhost", 42424, new StringToByteSerializer()));
-//
-// env.execute();
-// }
-//
-// public static class StringToByteSerializer implements SerializationSchema<String, byte[]> {
-//
-// private static final long serialVersionUID = 1L;
-//
-// @Override
-// public byte[] serialize(String element) {
-// return element.getBytes();
-// }
-// }
-//
-//}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
deleted file mode 100644
index 0f16541..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.json;
-
-import org.apache.flink.api.common.functions.RichFlatMapFunction;
-import org.apache.sling.commons.json.JSONException;
-
-/**
- * Abstract class derived from {@link RichFlatMapFunction} to handle JSON files.
- *
- * @param <IN>
- * Type of the input elements.
- * @param <OUT>
- * Type of the returned elements.
- */
-public abstract class JSONParseFlatMap<IN, OUT> extends RichFlatMapFunction<IN, OUT> {
-
- private static final long serialVersionUID = 1L;
-
- // private static final Log LOG = LogFactory.getLog(JSONParseFlatMap.class);
-
- /**
- * Get the value object associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public Object get(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).get("retValue");
- }
-
- /**
- * Get the boolean value associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public boolean getBoolean(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).getBoolean("retValue");
- }
-
- /**
- * Get the double value associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public double getDouble(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).getDouble("retValue");
- }
-
- /**
- * Get the int value associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public int getInt(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).getInt("retValue");
- }
-
- /**
- * Get the long value associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public long getLong(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).getLong("retValue");
- }
-
- /**
- * Get the String value associated with a key form a JSON code. It can find
- * embedded fields, too.
- *
- * @param jsonText
- * JSON String in which the field is searched.
- * @param field
- * The key whose value is searched for.
- * @return The object associated with the field.
- * @throws JSONException
- * If the field is not found.
- */
- public String getString(String jsonText, String field) throws JSONException {
- JSONParser parser = new JSONParser(jsonText);
-
- return parser.parse(field).getString("retValue");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
deleted file mode 100644
index c1eabbd..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.json;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.StringTokenizer;
-
-import org.apache.sling.commons.json.JSONArray;
-import org.apache.sling.commons.json.JSONException;
-import org.apache.sling.commons.json.JSONObject;
-
-/**
- * A JSONParser contains a JSONObject and provides opportunity to access
- * embedded fields in JSON code.
- */
-public class JSONParser {
-
- private JSONObject originalJO;
- private String searchedfield;
- private Object temp;
-
- /**
- * Construct a JSONParser from a string. The string has to be a JSON code
- * from which we want to get a field.
- *
- * @param jsonText
- * A string which contains a JSON code. String representation of
- * a JSON code.
- * @throws JSONException
- * If there is a syntax error in the source string.
- */
- public JSONParser(String jsonText) throws JSONException {
- originalJO = new JSONObject(jsonText);
- }
-
- /**
- *
- * Parse the JSON code passed to the constructor to find the given key.
- *
- * @param key
- * The key whose value is searched for.
- * @return A JSONObject which has only one field called "retValue" and the
- * value associated to it is the searched value. The methods of
- * JSONObject can be used to get the field value in a desired
- * format.
- * @throws JSONException
- * If the key is not found.
- */
- public JSONObject parse(String key) throws JSONException {
- initializeParser(key);
- parsing();
- return putResultInJSONObj();
- }
-
- /**
- * Prepare the fields of the class for the parsing
- *
- * @param key
- * The key whose value is searched for.
- * @throws JSONException
- * If the key is not found.
- */
- private void initializeParser(String key) throws JSONException {
- searchedfield = key;
- temp = new JSONObject(originalJO.toString());
- }
-
- /**
- * This function goes through the given field and calls the appropriate
- * functions to treat the units between the punctuation marks.
- *
- * @throws JSONException
- * If the key is not found.
- */
- private void parsing() throws JSONException {
- StringTokenizer st = new StringTokenizer(searchedfield, ".");
- while (st.hasMoreTokens()) {
- find(st.nextToken());
- }
- }
-
- /**
- * Search for the next part of the field and update the state if it was
- * found.
- *
- * @param nextToken
- * The current part of the searched field.
- * @throws JSONException
- * If the key is not found.
- */
- private void find(String nextToken) throws JSONException {
- if (endsWithBracket(nextToken)) {
- treatAllBracket(nextToken);
- } else {
- temp = ((JSONObject) temp).get(nextToken);
- }
- }
-
- /**
- * Determine whether the given string ends with a closing square bracket ']'
- *
- * @param nextToken
- * The current part of the searched field.
- * @return True if the given string ends with a closing square bracket ']'
- * and false otherwise.
- */
- private boolean endsWithBracket(String nextToken) {
- return nextToken.substring(nextToken.length() - 1).endsWith("]");
- }
-
- /**
- * Handle (multidimensional) arrays. Treat the square bracket pairs one
- * after the other if necessary.
- *
- * @param nextToken
- * The current part of the searched field.
- * @throws JSONException
- * If the searched element is not found.
- */
- private void treatAllBracket(String nextToken) throws JSONException {
- List<String> list = Arrays.asList(nextToken.split("\\["));
- ListIterator<String> iter = list.listIterator();
-
- temp = ((JSONObject) temp).get(iter.next());
-
- while (iter.hasNext()) {
- int index = Integer.parseInt(cutBracket(iter.next()));
- temp = ((JSONArray) temp).get(index);
- }
- }
-
- /**
- * Remove the last character of the string.
- *
- * @param string
- * String to modify.
- * @return The given string without the last character.
- */
- private String cutBracket(String string) {
- return string.substring(0, string.length() - 1);
- }
-
- /**
- * Save the result of the search into a JSONObject.
- *
- * @return A special JSONObject which contain only one key. The value
- * associated to this key is the result of the search.
- * @throws JSONException
- * If there is a problem creating the JSONObject. (e.g. invalid
- * syntax)
- */
- private JSONObject putResultInJSONObj() throws JSONException {
- JSONObject jo = new JSONObject();
- jo.put("retValue", temp);
- return jo;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java
deleted file mode 100644
index fe6684d..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.kafka.api.KafkaSource;
-import org.apache.flink.streaming.util.serialization.JavaDefaultStringSchema;
-
-public class KafkaConsumerExample {
-
- private static String host;
- private static int port;
- private static String topic;
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(4);
-
- DataStream<String> kafkaStream = env
- .addSource(new KafkaSource<String>(host + ":" + port, topic, new JavaDefaultStringSchema()));
- kafkaStream.print();
-
- env.execute();
- }
-
- private static boolean parseParameters(String[] args) {
- if (args.length == 3) {
- host = args[0];
- port = Integer.parseInt(args[1]);
- topic = args[2];
- return true;
- } else {
- System.err.println("Usage: KafkaConsumerExample <host> <port> <topic>");
- return false;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerExample.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerExample.java
deleted file mode 100644
index 4dd5577..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerExample.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.connectors.kafka.api.KafkaSink;
-import org.apache.flink.streaming.util.serialization.JavaDefaultStringSchema;
-
-public class KafkaProducerExample {
-
- private static String host;
- private static int port;
- private static String topic;
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(4);
-
- @SuppressWarnings({ "unused", "serial" })
- DataStream<String> stream1 = env.addSource(new SourceFunction<String>() {
-
- private int index = 0;
-
- @Override
- public boolean reachedEnd() throws Exception {
- return index >= 20;
- }
-
- @Override
- public String next() throws Exception {
- if (index < 20) {
- String result = "message #" + index;
- index++;
- return result;
- }
-
- return "q";
- }
-
- }).addSink(
- new KafkaSink<String>(host + ":" + port, topic, new JavaDefaultStringSchema())
- )
- .setParallelism(3);
-
- env.execute();
- }
-
- private static boolean parseParameters(String[] args) {
- if (args.length == 3) {
- host = args[0];
- port = Integer.parseInt(args[1]);
- topic = args[2];
- return true;
- } else {
- System.err.println("Usage: KafkaProducerExample <host> <port> <topic>");
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java
deleted file mode 100644
index 4286196..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.api.java.typeutils.runtime.ByteArrayInputView;
-import org.apache.flink.runtime.util.DataOutputSerializer;
-import org.apache.flink.streaming.util.serialization.DeserializationSchema;
-import org.apache.flink.streaming.util.serialization.SerializationSchema;
-
-import java.io.IOException;
-
-public class Utils {
- public static class TypeInformationSerializationSchema<T>
- implements DeserializationSchema<T>, SerializationSchema<T, byte[]> {
- private final TypeSerializer<T> serializer;
- private final TypeInformation<T> ti;
-
- public TypeInformationSerializationSchema(Object type, ExecutionConfig ec) {
- this.ti = (TypeInformation<T>) TypeExtractor.getForObject(type);
- this.serializer = ti.createSerializer(ec);
- }
- @Override
- public T deserialize(byte[] message) {
- try {
- return serializer.deserialize(new ByteArrayInputView(message));
- } catch (IOException e) {
- throw new RuntimeException("Unable to deserialize message", e);
- }
- }
-
- @Override
- public boolean isEndOfStream(T nextElement) {
- return false;
- }
-
- @Override
- public byte[] serialize(T element) {
- DataOutputSerializer dos = new DataOutputSerializer(16);
- try {
- serializer.serialize(element, dos);
- } catch (IOException e) {
- throw new RuntimeException("Unable to serialize record", e);
- }
- return dos.getByteArray();
- }
-
- @Override
- public TypeInformation<T> getProducedType() {
- return ti;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java
deleted file mode 100644
index 0965b29..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api;
-
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.flink.api.java.ClosureCleaner;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
-import org.apache.flink.streaming.connectors.kafka.api.config.PartitionerWrapper;
-import org.apache.flink.streaming.connectors.kafka.partitioner.SerializableKafkaPartitioner;
-import org.apache.flink.streaming.util.serialization.SerializationSchema;
-import org.apache.flink.util.NetUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-import kafka.javaapi.producer.Producer;
-import kafka.producer.KeyedMessage;
-import kafka.producer.ProducerConfig;
-import kafka.serializer.DefaultEncoder;
-
-
-/**
- * Sink that emits its inputs to a Kafka topic.
- *
- * @param <IN>
- * Type of the sink input
- */
-public class KafkaSink<IN> extends RichSinkFunction<IN> {
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaSink.class);
-
- private Producer<IN, byte[]> producer;
- private Properties userDefinedProperties;
- private String topicId;
- private String brokerList;
- private SerializationSchema<IN, byte[]> schema;
- private SerializableKafkaPartitioner partitioner;
- private Class<? extends SerializableKafkaPartitioner> partitionerClass = null;
-
- /**
- * Creates a KafkaSink for a given topic. The sink produces its input to
- * the topic.
- *
- * @param brokerList
- * Addresses of the brokers
- * @param topicId
- * ID of the Kafka topic.
- * @param serializationSchema
- * User defined serialization schema.
- */
- public KafkaSink(String brokerList, String topicId,
- SerializationSchema<IN, byte[]> serializationSchema) {
- this(brokerList, topicId, new Properties(), serializationSchema);
- }
-
- /**
- * Creates a KafkaSink for a given topic with custom Producer configuration.
- * If you use this constructor, the broker should be set with the "metadata.broker.list"
- * configuration.
- *
- * @param brokerList
- * Addresses of the brokers
- * @param topicId
- * ID of the Kafka topic.
- * @param producerConfig
- * Configurations of the Kafka producer
- * @param serializationSchema
- * User defined serialization schema.
- */
- public KafkaSink(String brokerList, String topicId, Properties producerConfig,
- SerializationSchema<IN, byte[]> serializationSchema) {
- String[] elements = brokerList.split(",");
- for(String broker: elements) {
- NetUtils.ensureCorrectHostnamePort(broker);
- }
- Preconditions.checkNotNull(topicId, "TopicID not set");
-
- this.brokerList = brokerList;
- this.topicId = topicId;
- this.schema = serializationSchema;
- this.partitionerClass = null;
- this.userDefinedProperties = producerConfig;
- }
-
- /**
- * Creates a KafkaSink for a given topic. The sink produces its input to
- * the topic.
- *
- * @param brokerList
- * @param topicId
- * ID of the Kafka topic.
- * @param serializationSchema
- * User defined serialization schema.
- * @param partitioner
- * User defined partitioner.
- */
- public KafkaSink(String brokerList, String topicId,
- SerializationSchema<IN, byte[]> serializationSchema, SerializableKafkaPartitioner partitioner) {
- this(brokerList, topicId, serializationSchema);
- ClosureCleaner.ensureSerializable(partitioner);
- this.partitioner = partitioner;
- }
-
- public KafkaSink(String brokerList,
- String topicId,
- SerializationSchema<IN, byte[]> serializationSchema,
- Class<? extends SerializableKafkaPartitioner> partitioner) {
- this(brokerList, topicId, serializationSchema);
- this.partitionerClass = partitioner;
- }
-
- /**
- * Initializes the connection to Kafka.
- */
- @Override
- public void open(Configuration configuration) {
-
- Properties properties = new Properties();
-
- properties.put("metadata.broker.list", brokerList);
- properties.put("request.required.acks", "-1");
- properties.put("message.send.max.retries", "10");
-
- properties.put("serializer.class", DefaultEncoder.class.getCanonicalName());
-
- // this will not be used as the key will not be serialized
- properties.put("key.serializer.class", DefaultEncoder.class.getCanonicalName());
-
- for (Map.Entry<Object, Object> propertiesEntry : userDefinedProperties.entrySet()) {
- properties.put(propertiesEntry.getKey(), propertiesEntry.getValue());
- }
-
- if (partitioner != null) {
- properties.put("partitioner.class", PartitionerWrapper.class.getCanonicalName());
- // java serialization will do the rest.
- properties.put(PartitionerWrapper.SERIALIZED_WRAPPER_NAME, partitioner);
- }
- if (partitionerClass != null) {
- properties.put("partitioner.class", partitionerClass);
- }
-
- ProducerConfig config = new ProducerConfig(properties);
-
- try {
- producer = new Producer<IN, byte[]>(config);
- } catch (NullPointerException e) {
- throw new RuntimeException("Cannot connect to Kafka broker " + brokerList, e);
- }
- }
-
- /**
- * Called when new data arrives to the sink, and forwards it to Kafka.
- *
- * @param next
- * The incoming data
- */
- @Override
- public void invoke(IN next) {
- byte[] serialized = schema.serialize(next);
-
- // Sending message without serializable key.
- producer.send(new KeyedMessage<IN, byte[]>(topicId, null, next, serialized));
- }
-
- @Override
- public void close() {
- if (producer != null) {
- producer.close();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java
deleted file mode 100644
index 00666f6..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import com.google.common.base.Preconditions;
-import kafka.consumer.Consumer;
-import kafka.consumer.ConsumerConfig;
-import kafka.consumer.ConsumerIterator;
-import kafka.consumer.KafkaStream;
-import kafka.javaapi.consumer.ConsumerConnector;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.connectors.ConnectorSource;
-import org.apache.flink.streaming.util.serialization.DeserializationSchema;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Source that listens to a Kafka topic using the high level Kafka API.
- *
- * @param <OUT>
- * Type of the messages on the topic.
- */
-public class KafkaSource<OUT> extends ConnectorSource<OUT> {
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaSource.class);
-
- private final String zookeeperAddress;
- private final String groupId;
- private final String topicId;
- private Properties customProperties;
-
- private transient ConsumerConnector consumer;
- private transient ConsumerIterator<byte[], byte[]> consumerIterator;
-
- private long zookeeperSyncTimeMillis;
- private static final long ZOOKEEPER_DEFAULT_SYNC_TIME = 200;
- private static final String DEFAULT_GROUP_ID = "flink-group";
-
- // We must read this in reachedEnd() to check for the end. We keep it to return it in
- // next()
- private OUT nextElement;
-
- /**
- * Creates a KafkaSource that consumes a topic.
- *
- * @param zookeeperAddress
- * Address of the Zookeeper host (with port number).
- * @param topicId
- * ID of the Kafka topic.
- * @param groupId
- * ID of the consumer group.
- * @param deserializationSchema
- * User defined deserialization schema.
- * @param zookeeperSyncTimeMillis
- * Synchronization time with zookeeper.
- */
- public KafkaSource(String zookeeperAddress,
- String topicId, String groupId,
- DeserializationSchema<OUT> deserializationSchema,
- long zookeeperSyncTimeMillis) {
- this(zookeeperAddress, topicId, groupId, deserializationSchema, zookeeperSyncTimeMillis, null);
- }
- /**
- * Creates a KafkaSource that consumes a topic.
- *
- * @param zookeeperAddress
- * Address of the Zookeeper host (with port number).
- * @param topicId
- * ID of the Kafka topic.
- * @param groupId
- * ID of the consumer group.
- * @param deserializationSchema
- * User defined deserialization schema.
- * @param zookeeperSyncTimeMillis
- * Synchronization time with zookeeper.
- * @param customProperties
- * Custom properties for Kafka
- */
- public KafkaSource(String zookeeperAddress,
- String topicId, String groupId,
- DeserializationSchema<OUT> deserializationSchema,
- long zookeeperSyncTimeMillis, Properties customProperties) {
- super(deserializationSchema);
- Preconditions.checkNotNull(zookeeperAddress, "ZK address is null");
- Preconditions.checkNotNull(topicId, "Topic ID is null");
- Preconditions.checkNotNull(deserializationSchema, "deserializationSchema is null");
- Preconditions.checkArgument(zookeeperSyncTimeMillis >= 0, "The ZK sync time must be positive");
-
- this.zookeeperAddress = zookeeperAddress;
- this.groupId = groupId;
- this.topicId = topicId;
- this.zookeeperSyncTimeMillis = zookeeperSyncTimeMillis;
- this.customProperties = customProperties;
- }
-
- /**
- * Creates a KafkaSource that consumes a topic.
- *
- * @param zookeeperAddress
- * Address of the Zookeeper host (with port number).
- * @param topicId
- * ID of the Kafka topic.
- * @param deserializationSchema
- * User defined deserialization schema.
- * @param zookeeperSyncTimeMillis
- * Synchronization time with zookeeper.
- */
- public KafkaSource(String zookeeperAddress, String topicId, DeserializationSchema<OUT> deserializationSchema, long zookeeperSyncTimeMillis) {
- this(zookeeperAddress, topicId, DEFAULT_GROUP_ID, deserializationSchema, zookeeperSyncTimeMillis, null);
- }
- /**
- * Creates a KafkaSource that consumes a topic.
- *
- * @param zookeeperAddress
- * Address of the Zookeeper host (with port number).
- * @param topicId
- * ID of the Kafka topic.
- * @param deserializationSchema
- * User defined deserialization schema.
- */
- public KafkaSource(String zookeeperAddress, String topicId, DeserializationSchema<OUT> deserializationSchema) {
- this(zookeeperAddress, topicId, deserializationSchema, ZOOKEEPER_DEFAULT_SYNC_TIME);
- }
-
- /**
- * Initializes the connection to Kafka.
- */
- private void initializeConnection() {
- Properties props = new Properties();
- props.put("zookeeper.connect", zookeeperAddress);
- props.put("group.id", groupId);
- props.put("zookeeper.session.timeout.ms", "10000");
- props.put("zookeeper.sync.time.ms", Long.toString(zookeeperSyncTimeMillis));
- props.put("auto.commit.interval.ms", "1000");
-
- if(customProperties != null) {
- for(Map.Entry<Object, Object> e : props.entrySet()) {
- if(props.contains(e.getKey())) {
- LOG.warn("Overwriting property "+e.getKey()+" with value "+e.getValue());
- }
- props.put(e.getKey(), e.getValue());
- }
- }
-
- consumer = Consumer.createJavaConsumerConnector(new ConsumerConfig(props));
-
- Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer
- .createMessageStreams(Collections.singletonMap(topicId, 1));
- List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topicId);
- KafkaStream<byte[], byte[]> stream = streams.get(0);
-
- consumer.commitOffsets();
-
- consumerIterator = stream.iterator();
- }
-
- @Override
- public void open(Configuration config) throws Exception {
- super.open(config);
- initializeConnection();
- }
-
- @Override
- public void close() throws Exception {
- super.close();
- if (consumer != null) {
- consumer.shutdown();
- }
- }
-
- @Override
- public boolean reachedEnd() throws Exception {
- if (nextElement != null) {
- return false;
- } else if (consumerIterator.hasNext()) {
- OUT out = schema.deserialize(consumerIterator.next().message());
- if (schema.isEndOfStream(out)) {
- return true;
- }
- nextElement = out;
- }
- return false;
- }
-
- @Override
- public OUT next() throws Exception {
- if (!reachedEnd()) {
- OUT result = nextElement;
- nextElement = null;
- return result;
- } else {
- throw new RuntimeException("Source exhausted");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/config/PartitionerWrapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/config/PartitionerWrapper.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/config/PartitionerWrapper.java
deleted file mode 100644
index 7ae17df..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/config/PartitionerWrapper.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.streaming.connectors.kafka.api.config;
-
-import kafka.producer.Partitioner;
-import kafka.utils.VerifiableProperties;
-
-/**
- * Hacky wrapper to send an object instance through a Properties - map.
- *
- * This works as follows:
- * The recommended way of creating a KafkaSink is specifying a classname for the partitioner.
- *
- * Otherwise (if the user gave a (serializable) class instance), we give Kafka the PartitionerWrapper class of Flink.
- * This is set in the key-value (java.util.Properties) map.
- * In addition to that, we use the Properties.put(Object, Object) to store the instance of the (serializable).
- * This is a hack because the put() method is called on the underlying Hashmap.
- *
- * This PartitionerWrapper is called with the Properties. From there, we extract the wrapped Partitioner instance.
- *
- * The serializable PartitionerWrapper is serialized into the Properties Hashmap and also deserialized from there.
- */
-public class PartitionerWrapper implements Partitioner {
- public final static String SERIALIZED_WRAPPER_NAME = "flink.kafka.wrapper.serialized";
-
- private Partitioner wrapped;
- public PartitionerWrapper(VerifiableProperties properties) {
- wrapped = (Partitioner) properties.props().get(SERIALIZED_WRAPPER_NAME);
- }
-
- @Override
- public int partition(Object value, int numberOfPartitions) {
- return wrapped.partition(value, numberOfPartitions);
- }
-}
[4/7] flink git commit: [FLINK-1874] [streaming] Connectors separated
into individual projects
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/kafka/KafkaITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/kafka/KafkaITCase.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/kafka/KafkaITCase.java
new file mode 100644
index 0000000..7b7bdcc
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/kafka/KafkaITCase.java
@@ -0,0 +1,1023 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import kafka.admin.AdminUtils;
+import kafka.api.PartitionMetadata;
+import kafka.consumer.Consumer;
+import kafka.consumer.ConsumerConfig;
+import kafka.consumer.ConsumerIterator;
+import kafka.consumer.KafkaStream;
+import kafka.javaapi.consumer.ConsumerConnector;
+import kafka.message.MessageAndMetadata;
+import kafka.network.SocketServer;
+import org.I0Itec.zkclient.ZkClient;
+import org.apache.curator.test.TestingServer;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.net.NetUtils;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.flink.streaming.api.functions.sink.SinkFunction;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.connectors.kafka.api.KafkaSink;
+import org.apache.flink.streaming.connectors.kafka.api.KafkaSource;
+import org.apache.flink.streaming.connectors.kafka.api.persistent.PersistentKafkaSource;
+import org.apache.flink.streaming.connectors.kafka.partitioner.SerializableKafkaPartitioner;
+import org.apache.flink.streaming.connectors.kafka.util.KafkaLocalSystemTime;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.flink.streaming.util.serialization.JavaDefaultStringSchema;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.StringUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import kafka.server.KafkaConfig;
+import kafka.server.KafkaServer;
+import scala.collection.Seq;
+
+/**
+ * Code in this test is based on the following GitHub repository:
+ * (as per commit bc6b2b2d5f6424d5f377aa6c0871e82a956462ef)
+ * <p/>
+ * https://github.com/sakserv/hadoop-mini-clusters (ASL licensed)
+ */
+
+public class KafkaITCase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaITCase.class);
+ private static final int NUMBER_OF_KAFKA_SERVERS = 3;
+
+ private static int zkPort;
+ private static String kafkaHost;
+
+ private static String zookeeperConnectionString;
+
+ @ClassRule
+ public static TemporaryFolder tempFolder = new TemporaryFolder();
+ public static File tmpZkDir;
+ public static List<File> tmpKafkaDirs;
+
+ private static TestingServer zookeeper;
+ private static List<KafkaServer> brokers;
+ private static String brokerConnectionStrings = "";
+
+ private static ConsumerConfig standardCC;
+
+ private static ZkClient zkClient;
+
+
+ @BeforeClass
+ public static void prepare() throws IOException {
+ LOG.info("Starting KafkaITCase.prepare()");
+ tmpZkDir = tempFolder.newFolder();
+
+ tmpKafkaDirs = new ArrayList<File>(NUMBER_OF_KAFKA_SERVERS);
+ for (int i = 0; i < NUMBER_OF_KAFKA_SERVERS; i++) {
+ tmpKafkaDirs.add(tempFolder.newFolder());
+ }
+
+ kafkaHost = InetAddress.getLocalHost().getHostName();
+ zkPort = NetUtils.getAvailablePort();
+ zookeeperConnectionString = "localhost:" + zkPort;
+
+ zookeeper = null;
+ brokers = null;
+
+ try {
+ LOG.info("Starting Zookeeper");
+ zookeeper = getZookeeper();
+ LOG.info("Starting KafkaServer");
+ brokers = new ArrayList<KafkaServer>(NUMBER_OF_KAFKA_SERVERS);
+ for (int i = 0; i < NUMBER_OF_KAFKA_SERVERS; i++) {
+ brokers.add(getKafkaServer(i, tmpKafkaDirs.get(i)));
+ SocketServer socketServer = brokers.get(i).socketServer();
+ String host = "localhost";
+ if(socketServer.host() != null) {
+ host = socketServer.host();
+ }
+ brokerConnectionStrings += host+":"+socketServer.port()+",";
+ }
+
+ LOG.info("ZK and KafkaServer started.");
+ } catch (Throwable t) {
+ LOG.warn("Test failed with exception", t);
+ Assert.fail("Test failed with: " + t.getMessage());
+ }
+
+ Properties cProps = new Properties();
+ cProps.setProperty("zookeeper.connect", zookeeperConnectionString);
+ cProps.setProperty("group.id", "flink-tests");
+ cProps.setProperty("auto.commit.enable", "false");
+
+ cProps.setProperty("auto.offset.reset", "smallest"); // read from the beginning.
+
+ standardCC = new ConsumerConfig(cProps);
+
+ zkClient = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(), standardCC.zkConnectionTimeoutMs(), new PersistentKafkaSource.KafkaZKStringSerializer());
+ }
+
+ @AfterClass
+ public static void shutDownServices() {
+ LOG.info("Shutting down all services");
+ for (KafkaServer broker : brokers) {
+ if (broker != null) {
+ broker.shutdown();
+ }
+ }
+ if (zookeeper != null) {
+ try {
+ zookeeper.stop();
+ } catch (IOException e) {
+ LOG.warn("ZK.stop() failed", e);
+ }
+ }
+ zkClient.close();
+ }
+
+
+ @Test
+ public void testOffsetManipulation() {
+ ZkClient zk = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(), standardCC.zkConnectionTimeoutMs(), new PersistentKafkaSource.KafkaZKStringSerializer());
+
+ final String topicName = "testOffsetManipulation";
+
+ // create topic
+ Properties topicConfig = new Properties();
+ LOG.info("Creating topic {}", topicName);
+ AdminUtils.createTopic(zk, topicName, 3, 2, topicConfig);
+
+ PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 0, 1337);
+
+ Assert.assertEquals(1337L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 0));
+
+ zk.close();
+ }
+ /**
+ * We want to use the High level java consumer API but manage the offset in Zookeeper manually.
+ *
+ */
+ @Test
+ public void testPersistentSourceWithOffsetUpdates() throws Exception {
+ LOG.info("Starting testPersistentSourceWithOffsetUpdates()");
+
+ ZkClient zk = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(), standardCC.zkConnectionTimeoutMs(), new PersistentKafkaSource.KafkaZKStringSerializer());
+
+ final String topicName = "testOffsetHacking";
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(3);
+ env.getConfig().disableSysoutLogging();
+ env.enableCheckpointing(50);
+ env.setNumberOfExecutionRetries(0);
+
+ // create topic
+ Properties topicConfig = new Properties();
+ LOG.info("Creating topic {}", topicName);
+ AdminUtils.createTopic(zk, topicName, 3, 2, topicConfig);
+
+ // write a sequence from 0 to 99 to each of the three partitions.
+ writeSequence(env, topicName, 0, 99);
+
+
+ readSequence(env, standardCC, topicName, 0, 100, 300);
+
+ // check offsets to be set at least higher than 50.
+ // correctly, we would expect them to be set to 99, but right now there is no way of stopping a topology once all pending
+ // checkpoints have been committed.
+ // To work around that limitation, the persistent kafka consumer is throtteled with a thread.sleep().
+ long o1 = PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 0);
+ long o2 = PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 1);
+ long o3 = PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 2);
+ Assert.assertTrue("The offset seems incorrect, got "+o1, o1 > 50L);
+ Assert.assertTrue("The offset seems incorrect, got "+o2, o2 > 50L);
+ Assert.assertTrue("The offset seems incorrect, got "+o3, o3 > 50L);
+ /** Once we have proper shutdown of streaming jobs, enable these tests
+ Assert.assertEquals("The offset seems incorrect", 99L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 0));
+ Assert.assertEquals("The offset seems incorrect", 99L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 1));
+ Assert.assertEquals("The offset seems incorrect", 99L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 2));*/
+
+
+ LOG.info("Manipulating offsets");
+ // set the offset to 25, 50, and 75 for the three partitions
+ PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 0, 50);
+ PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 1, 50);
+ PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 2, 50);
+
+ // create new env
+ env = StreamExecutionEnvironment.createLocalEnvironment(3);
+ env.getConfig().disableSysoutLogging();
+ readSequence(env, standardCC, topicName, 50, 50, 150);
+
+ zk.close();
+
+ LOG.info("Finished testPersistentSourceWithOffsetUpdates()");
+ }
+
+ private void readSequence(StreamExecutionEnvironment env, ConsumerConfig cc, final String topicName, final int valuesStartFrom, final int valuesCount, final int finalCount) throws Exception {
+ LOG.info("Reading sequence for verification until final count {}", finalCount);
+ DataStream<Tuple2<Integer, Integer>> source = env.addSource(
+ new PersistentKafkaSource<Tuple2<Integer, Integer>>(topicName, new Utils.TypeInformationSerializationSchema<Tuple2<Integer, Integer>>(new Tuple2<Integer, Integer>(1,1), env.getConfig()), cc)
+ )
+ //add a sleeper mapper. Since there is no good way of "shutting down" a running topology, we have
+ // to play this trick. The problem is that we have to wait until all checkpoints are confirmed
+ .map(new MapFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>>() {
+ @Override
+ public Tuple2<Integer, Integer> map(Tuple2<Integer, Integer> value) throws Exception {
+ Thread.sleep(150);
+ return value;
+ }
+ }).setParallelism(3);
+
+ // verify data
+ DataStream<Integer> validIndexes = source.flatMap(new RichFlatMapFunction<Tuple2<Integer, Integer>, Integer>() {
+ int[] values = new int[valuesCount];
+ int count = 0;
+
+ @Override
+ public void flatMap(Tuple2<Integer, Integer> value, Collector<Integer> out) throws Exception {
+ values[value.f1 - valuesStartFrom]++;
+ count++;
+
+ LOG.info("Reader " + getRuntimeContext().getIndexOfThisSubtask() + " got " + value + " count=" + count + "/" + finalCount);
+ // verify if we've seen everything
+
+ if (count == finalCount) {
+ LOG.info("Received all values");
+ for (int i = 0; i < values.length; i++) {
+ int v = values[i];
+ if (v != 3) {
+ LOG.warn("Test is going to fail");
+ printTopic(topicName, valuesCount, this.getRuntimeContext().getExecutionConfig());
+ throw new RuntimeException("Expected v to be 3, but was " + v + " on element " + i + " array=" + Arrays.toString(values));
+ }
+ }
+ // test has passed
+ throw new SuccessException();
+ }
+ }
+
+ }).setParallelism(1);
+
+ tryExecute(env, "Read data from Kafka");
+
+ LOG.info("Successfully read sequence for verification");
+ }
+
+
+
+ private void writeSequence(StreamExecutionEnvironment env, String topicName, final int from, final int to) throws Exception {
+ LOG.info("Writing sequence from {} to {} to topic {}", from, to, topicName);
+
+ DataStream<Tuple2<Integer, Integer>> stream = env.addSource(new RichParallelSourceFunction<Tuple2<Integer, Integer>>() {
+ private static final long serialVersionUID = 1L;
+ int cnt = from;
+ int partition;
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ partition = getRuntimeContext().getIndexOfThisSubtask();
+
+ }
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ return cnt > to;
+ }
+
+ @Override
+ public Tuple2<Integer, Integer> next() throws Exception {
+ LOG.info("Writing " + cnt + " to partition " + partition);
+ Tuple2<Integer, Integer> result = new Tuple2<Integer, Integer>(getRuntimeContext().getIndexOfThisSubtask(), cnt);
+ cnt++;
+ return result;
+ }
+ }).setParallelism(3);
+
+ stream.addSink(new KafkaSink<Tuple2<Integer, Integer>>(brokerConnectionStrings,
+ topicName,
+ new Utils.TypeInformationSerializationSchema<Tuple2<Integer, Integer>>(new Tuple2<Integer, Integer>(1, 1), env.getConfig()),
+ new T2Partitioner()
+ )).setParallelism(3);
+ env.execute("Write sequence from " + from + " to " + to + " to topic " + topicName);
+ LOG.info("Finished writing sequence");
+ }
+
+ private static class T2Partitioner implements SerializableKafkaPartitioner {
+ @Override
+ public int partition(Object key, int numPartitions) {
+ if(numPartitions != 3) {
+ throw new IllegalArgumentException("Expected three partitions");
+ }
+ Tuple2<Integer, Integer> element = (Tuple2<Integer, Integer>) key;
+ return element.f0;
+ }
+ }
+
+
+ @Test
+ public void regularKafkaSourceTest() throws Exception {
+ LOG.info("Starting KafkaITCase.regularKafkaSourceTest()");
+
+ String topic = "regularKafkaSourceTestTopic";
+ createTestTopic(topic, 1, 1);
+
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+ // add consuming topology:
+ DataStreamSource<Tuple2<Long, String>> consuming = env.addSource(
+ new KafkaSource<Tuple2<Long, String>>(zookeeperConnectionString, topic, "myFlinkGroup", new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()), 5000));
+ consuming.addSink(new SinkFunction<Tuple2<Long, String>>() {
+ int elCnt = 0;
+ int start = -1;
+ BitSet validator = new BitSet(101);
+
+ @Override
+ public void invoke(Tuple2<Long, String> value) throws Exception {
+ LOG.debug("Got value = " + value);
+ String[] sp = value.f1.split("-");
+ int v = Integer.parseInt(sp[1]);
+
+ assertEquals(value.f0 - 1000, (long) v);
+
+ if (start == -1) {
+ start = v;
+ }
+ Assert.assertFalse("Received tuple twice", validator.get(v - start));
+ validator.set(v - start);
+ elCnt++;
+ if (elCnt == 100) {
+ // check if everything in the bitset is set to true
+ int nc;
+ if ((nc = validator.nextClearBit(0)) != 100) {
+ throw new RuntimeException("The bitset was not set to 1 on all elements. Next clear:" + nc + " Set: " + validator);
+ }
+ throw new SuccessException();
+ }
+ }
+ });
+
+ // add producing topology
+ DataStream<Tuple2<Long, String>> stream = env.addSource(new SourceFunction<Tuple2<Long, String>>() {
+ private static final long serialVersionUID = 1L;
+ int cnt = 0;
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ return false;
+ }
+
+ @Override
+ public Tuple2<Long, String> next() throws Exception {
+ Thread.sleep(100);
+ return new Tuple2<Long, String>(1000L + cnt, "kafka-" + cnt++);
+ }
+ });
+ stream.addSink(new KafkaSink<Tuple2<Long, String>>(brokerConnectionStrings, topic, new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig())));
+
+ tryExecute(env, "regular kafka source test");
+
+ LOG.info("Finished KafkaITCase.regularKafkaSourceTest()");
+ }
+
+ @Test
+ public void tupleTestTopology() throws Exception {
+ LOG.info("Starting KafkaITCase.tupleTestTopology()");
+
+ String topic = "tupleTestTopic";
+ createTestTopic(topic, 1, 1);
+
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+
+ // add consuming topology:
+ DataStreamSource<Tuple2<Long, String>> consuming = env.addSource(
+ new PersistentKafkaSource<Tuple2<Long, String>>(topic,
+ new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()),
+ standardCC
+ ));
+ consuming.addSink(new RichSinkFunction<Tuple2<Long, String>>() {
+ int elCnt = 0;
+ int start = -1;
+ BitSet validator = new BitSet(101);
+
+ @Override
+ public void invoke(Tuple2<Long, String> value) throws Exception {
+ LOG.info("Got value " + value);
+ String[] sp = value.f1.split("-");
+ int v = Integer.parseInt(sp[1]);
+
+ assertEquals(value.f0 - 1000, (long) v);
+
+ if (start == -1) {
+ start = v;
+ }
+ Assert.assertFalse("Received tuple twice", validator.get(v - start));
+ validator.set(v - start);
+ elCnt++;
+ if (elCnt == 100) {
+ // check if everything in the bitset is set to true
+ int nc;
+ if ((nc = validator.nextClearBit(0)) != 100) {
+ throw new RuntimeException("The bitset was not set to 1 on all elements. Next clear:" + nc + " Set: " + validator);
+ }
+ throw new SuccessException();
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ Assert.assertTrue("No element received", elCnt > 0);
+ }
+ });
+
+ // add producing topology
+ DataStream<Tuple2<Long, String>> stream = env.addSource(new SourceFunction<Tuple2<Long, String>>() {
+ private static final long serialVersionUID = 1L;
+ int cnt = 0;
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ return false;
+ }
+
+ @Override
+ public Tuple2<Long, String> next() throws Exception {
+ Thread.sleep(100);
+ return new Tuple2<Long, String>(1000L + cnt, "kafka-" + cnt++);
+ }
+ });
+ stream.addSink(new KafkaSink<Tuple2<Long, String>>(brokerConnectionStrings, topic, new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig())));
+
+ tryExecute(env, "tupletesttopology");
+
+ LOG.info("Finished KafkaITCase.tupleTestTopology()");
+ }
+
+ /**
+ * Test Flink's Kafka integration also with very big records (30MB)
+ *
+ * see http://stackoverflow.com/questions/21020347/kafka-sending-a-15mb-message
+ *
+ * @throws Exception
+ */
+ @Test
+ public void bigRecordTestTopology() throws Exception {
+
+ LOG.info("Starting KafkaITCase.bigRecordTestTopology()");
+
+ String topic = "bigRecordTestTopic";
+ createTestTopic(topic, 1, 1);
+
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+
+ // add consuming topology:
+ Utils.TypeInformationSerializationSchema<Tuple2<Long, byte[]>> serSchema = new Utils.TypeInformationSerializationSchema<Tuple2<Long, byte[]>>(new Tuple2<Long, byte[]>(0L, new byte[]{0}), env.getConfig());
+ Properties consumerProps = new Properties();
+ consumerProps.setProperty("fetch.message.max.bytes", Integer.toString(1024 * 1024 * 30));
+ consumerProps.setProperty("zookeeper.connect", zookeeperConnectionString);
+ consumerProps.setProperty("group.id", "test");
+ consumerProps.setProperty("auto.commit.enable", "false");
+ consumerProps.setProperty("auto.offset.reset", "smallest");
+
+ ConsumerConfig cc = new ConsumerConfig(consumerProps);
+ DataStreamSource<Tuple2<Long, byte[]>> consuming = env.addSource(
+ new PersistentKafkaSource<Tuple2<Long, byte[]>>(topic, serSchema, cc));
+
+ consuming.addSink(new SinkFunction<Tuple2<Long, byte[]>>() {
+ int elCnt = 0;
+
+ @Override
+ public void invoke(Tuple2<Long, byte[]> value) throws Exception {
+ LOG.info("Received {}", value.f0);
+ elCnt++;
+ if(value.f0 == -1) {
+ // we should have seen 11 elements now.
+ if(elCnt == 11) {
+ throw new SuccessException();
+ } else {
+ throw new RuntimeException("There have been "+elCnt+" elements");
+ }
+ }
+ if(elCnt > 10) {
+ throw new RuntimeException("More than 10 elements seen: "+elCnt);
+ }
+ }
+ }).setParallelism(1);
+
+ // add producing topology
+ DataStream<Tuple2<Long, byte[]>> stream = env.addSource(new RichSourceFunction<Tuple2<Long, byte[]>>() {
+ private static final long serialVersionUID = 1L;
+ boolean running = true;
+ long cnt;
+ transient Random rnd;
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ cnt = 0;
+ rnd = new Random(1337);
+
+ }
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ return cnt > 10;
+ }
+
+ @Override
+ public Tuple2<Long, byte[]> next() throws Exception {
+ Thread.sleep(100);
+
+ if (cnt < 10) {
+ byte[] wl = new byte[Math.abs(rnd.nextInt(1024 * 1024 * 30))];
+ Tuple2<Long, byte[]> result = new Tuple2<Long, byte[]>(cnt++, wl);
+ LOG.info("Emitted cnt=" + (cnt - 1) + " with byte.length = " + wl.length);
+ return result;
+
+ } else if (cnt == 10) {
+ Tuple2<Long, byte[]> result = new Tuple2<Long, byte[]>(-1L, new byte[]{1});
+ cnt++;
+ return result;
+ } else {
+ throw new RuntimeException("Source is exhausted.");
+ }
+ }
+ });
+
+ stream.addSink(new KafkaSink<Tuple2<Long, byte[]>>(brokerConnectionStrings, topic,
+ new Utils.TypeInformationSerializationSchema<Tuple2<Long, byte[]>>(new Tuple2<Long, byte[]>(0L, new byte[]{0}), env.getConfig()))
+ );
+
+ tryExecute(env, "big topology test");
+
+ LOG.info("Finished KafkaITCase.bigRecordTestTopology()");
+ }
+
+
+ @Test
+ public void customPartitioningTestTopology() throws Exception {
+ LOG.info("Starting KafkaITCase.customPartitioningTestTopology()");
+
+ String topic = "customPartitioningTestTopic";
+
+ createTestTopic(topic, 3, 1);
+
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+
+ // add consuming topology:
+ DataStreamSource<Tuple2<Long, String>> consuming = env.addSource(
+ new PersistentKafkaSource<Tuple2<Long, String>>(topic,
+ new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()),
+ standardCC));
+ consuming.addSink(new SinkFunction<Tuple2<Long, String>>() {
+ int start = -1;
+ BitSet validator = new BitSet(101);
+
+ boolean gotPartition1 = false;
+ boolean gotPartition2 = false;
+ boolean gotPartition3 = false;
+
+ @Override
+ public void invoke(Tuple2<Long, String> value) throws Exception {
+ LOG.debug("Got " + value);
+ String[] sp = value.f1.split("-");
+ int v = Integer.parseInt(sp[1]);
+
+ assertEquals(value.f0 - 1000, (long) v);
+
+ switch (v) {
+ case 9:
+ gotPartition1 = true;
+ break;
+ case 19:
+ gotPartition2 = true;
+ break;
+ case 99:
+ gotPartition3 = true;
+ break;
+ }
+
+ if (start == -1) {
+ start = v;
+ }
+ Assert.assertFalse("Received tuple twice", validator.get(v - start));
+ validator.set(v - start);
+
+ if (gotPartition1 && gotPartition2 && gotPartition3) {
+ // check if everything in the bitset is set to true
+ int nc;
+ if ((nc = validator.nextClearBit(0)) != 100) {
+ throw new RuntimeException("The bitset was not set to 1 on all elements. Next clear:" + nc + " Set: " + validator);
+ }
+ throw new SuccessException();
+ }
+ }
+ });
+
+ // add producing topology
+ DataStream<Tuple2<Long, String>> stream = env.addSource(new SourceFunction<Tuple2<Long, String>>() {
+ private static final long serialVersionUID = 1L;
+ int cnt = 0;
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ return false;
+ }
+
+ @Override
+ public Tuple2<Long, String> next() throws Exception {
+ Thread.sleep(100);
+ return new Tuple2<Long, String>(1000L + cnt, "kafka-" + cnt++);
+ }
+ });
+ stream.addSink(new KafkaSink<Tuple2<Long, String>>(brokerConnectionStrings, topic, new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()), new CustomPartitioner()));
+
+ tryExecute(env, "custom partitioning test");
+
+ LOG.info("Finished KafkaITCase.customPartitioningTestTopology()");
+ }
+
+ /**
+ * This is for a topic with 3 partitions and Tuple2<Long, String>
+ */
+ private static class CustomPartitioner implements SerializableKafkaPartitioner {
+
+ @Override
+ public int partition(Object key, int numPartitions) {
+
+ @SuppressWarnings("unchecked")
+ Tuple2<Long, String> tuple = (Tuple2<Long, String>) key;
+ if (tuple.f0 < 10) {
+ return 0;
+ } else if (tuple.f0 < 20) {
+ return 1;
+ } else {
+ return 2;
+ }
+ }
+ }
+
+
+ @Test
+ public void simpleTestTopology() throws Exception {
+ String topic = "simpleTestTopic";
+
+ createTestTopic(topic, 1, 1);
+
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+
+ // add consuming topology:
+ DataStreamSource<String> consuming = env.addSource(
+ new PersistentKafkaSource<String>(topic, new JavaDefaultStringSchema(), standardCC));
+ consuming.addSink(new SinkFunction<String>() {
+ int elCnt = 0;
+ int start = -1;
+ BitSet validator = new BitSet(101);
+
+ @Override
+ public void invoke(String value) throws Exception {
+ LOG.debug("Got " + value);
+ String[] sp = value.split("-");
+ int v = Integer.parseInt(sp[1]);
+ if (start == -1) {
+ start = v;
+ }
+ Assert.assertFalse("Received tuple twice", validator.get(v - start));
+ validator.set(v - start);
+ elCnt++;
+ if (elCnt == 100) {
+ // check if everything in the bitset is set to true
+ int nc;
+ if ((nc = validator.nextClearBit(0)) != 100) {
+ throw new RuntimeException("The bitset was not set to 1 on all elements. Next clear:" + nc + " Set: " + validator);
+ }
+ throw new SuccessException();
+ }
+ }
+ });
+
+ // add producing topology
+ DataStream<String> stream = env.addSource(new SourceFunction<String>() {
+ private static final long serialVersionUID = 1L;
+ boolean running = true;
+ int cnt = 0;
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ return false;
+ }
+
+ @Override
+ public String next() throws Exception {
+ Thread.sleep(100);
+ return "kafka-" + cnt++;
+ }
+ });
+ stream.addSink(new KafkaSink<String>(brokerConnectionStrings, topic, new JavaDefaultStringSchema()));
+
+ tryExecute(env, "simpletest");
+ }
+
+ private static boolean leaderHasShutDown = false;
+ private static boolean shutdownKafkaBroker;
+
+ @Test(timeout=60000)
+ public void brokerFailureTest() throws Exception {
+ String topic = "brokerFailureTestTopic";
+
+ createTestTopic(topic, 2, 2);
+
+ // --------------------------- write data to topic ---------------------
+ LOG.info("Writing data to topic {}", topic);
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+
+ DataStream<String> stream = env.addSource(new SourceFunction<String>() {
+
+ private int cnt = 0;
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ return cnt == 200;
+ }
+
+ @Override
+ public String next() throws Exception {
+ String msg = "kafka-" + cnt++;
+ LOG.info("sending message = "+msg);
+
+ if ((cnt - 1) % 20 == 0) {
+ LOG.debug("Sending message #{}", cnt - 1);
+ }
+
+ return msg;
+ }
+
+ });
+ stream.addSink(new KafkaSink<String>(brokerConnectionStrings, topic, new JavaDefaultStringSchema()))
+ .setParallelism(1);
+
+ tryExecute(env, "broker failure test - writer");
+
+ // --------------------------- read and let broker fail ---------------------
+
+ LOG.info("Reading data from topic {} and let a broker fail", topic);
+ PartitionMetadata firstPart = null;
+ do {
+ if(firstPart != null) {
+ LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
+ // not the first try. Sleep a bit
+ Thread.sleep(150);
+ }
+ Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient).partitionsMetadata();
+ firstPart = partitionMetadata.head();
+ } while(firstPart.errorCode() != 0);
+
+ final String leaderToShutDown = firstPart.leader().get().connectionString();
+ LOG.info("Leader to shutdown {}", leaderToShutDown);
+
+ final Thread brokerShutdown = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ shutdownKafkaBroker = false;
+ while (!shutdownKafkaBroker) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ LOG.warn("Interruption", e);
+ }
+ }
+
+ for (KafkaServer kafkaServer : brokers) {
+ if (leaderToShutDown.equals(kafkaServer.config().advertisedHostName()+ ":"+ kafkaServer.config().advertisedPort())) {
+ LOG.info("Killing Kafka Server {}", leaderToShutDown);
+ kafkaServer.shutdown();
+ leaderHasShutDown = true;
+ break;
+ }
+ }
+ }
+ });
+ brokerShutdown.start();
+
+ // add consuming topology:
+ DataStreamSource<String> consuming = env.addSource(new PersistentKafkaSource<String>(topic, new JavaDefaultStringSchema(), standardCC));
+ consuming.setParallelism(1);
+
+ consuming.addSink(new SinkFunction<String>() {
+ int elCnt = 0;
+ int start = 0;
+ int numOfMessagesToBeCorrect = 100;
+ int stopAfterMessages = 150;
+
+ BitSet validator = new BitSet(numOfMessagesToBeCorrect + 1);
+
+ @Override
+ public void invoke(String value) throws Exception {
+ LOG.info("Got message = " + value + " leader has shut down " + leaderHasShutDown + " el cnt = " + elCnt + " to rec" + numOfMessagesToBeCorrect);
+ String[] sp = value.split("-");
+ int v = Integer.parseInt(sp[1]);
+
+ if (start == -1) {
+ start = v;
+ }
+ int offset = v - start;
+ Assert.assertFalse("Received tuple with value " + offset + " twice", validator.get(offset));
+ if (v - start < 0 && LOG.isWarnEnabled()) {
+ LOG.warn("Not in order: {}", value);
+ }
+
+ validator.set(offset);
+ elCnt++;
+ if (elCnt == 20) {
+ LOG.info("Asking leading broker to shut down");
+ // shut down a Kafka broker
+ shutdownKafkaBroker = true;
+ }
+ if (shutdownKafkaBroker) {
+ // we become a bit slower because the shutdown takes some time and we have
+ // only a fixed nubmer of elements to read
+ Thread.sleep(20);
+ }
+ if (leaderHasShutDown) { // it only makes sence to check once the shutdown is completed
+ if (elCnt >= stopAfterMessages) {
+ // check if everything in the bitset is set to true
+ int nc;
+ if ((nc = validator.nextClearBit(0)) < numOfMessagesToBeCorrect) {
+ throw new RuntimeException("The bitset was not set to 1 on all elements to be checked. Next clear:" + nc + " Set: " + validator);
+ }
+ throw new SuccessException();
+ }
+ }
+ }
+ });
+ tryExecute(env, "broker failure test - reader");
+
+ }
+
+ public static void tryExecute(StreamExecutionEnvironment see, String name) throws Exception {
+ try {
+ see.execute(name);
+ } catch (JobExecutionException good) {
+ Throwable t = good.getCause();
+ int limit = 0;
+ while (!(t instanceof SuccessException)) {
+ if(t == null) {
+ LOG.warn("Test failed with exception", good);
+ Assert.fail("Test failed with: " + StringUtils.stringifyException(good));
+ }
+
+ t = t.getCause();
+ if (limit++ == 20) {
+ LOG.warn("Test failed with exception", good);
+ Assert.fail("Test failed with: " + StringUtils.stringifyException(good));
+ }
+ }
+ }
+ }
+
+ private void createTestTopic(String topic, int numberOfPartitions, int replicationFactor) {
+ // create topic
+ Properties topicConfig = new Properties();
+ LOG.info("Creating topic {}", topic);
+ AdminUtils.createTopic(zkClient, topic, numberOfPartitions, replicationFactor, topicConfig);
+ }
+
+ private static TestingServer getZookeeper() throws Exception {
+ return new TestingServer(zkPort, tmpZkDir);
+ }
+
+ /**
+ * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed)
+ */
+ private static KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws UnknownHostException {
+ Properties kafkaProperties = new Properties();
+
+ int kafkaPort = NetUtils.getAvailablePort();
+
+ // properties have to be Strings
+ kafkaProperties.put("advertised.host.name", kafkaHost);
+ kafkaProperties.put("port", Integer.toString(kafkaPort));
+ kafkaProperties.put("broker.id", Integer.toString(brokerId));
+ kafkaProperties.put("log.dir", tmpFolder.toString());
+ kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
+ kafkaProperties.put("message.max.bytes", "" + (35 * 1024 * 1024));
+ kafkaProperties.put("replica.fetch.max.bytes", "" + (35 * 1024 * 1024));
+ KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);
+
+ KafkaServer server = new KafkaServer(kafkaConfig, new KafkaLocalSystemTime());
+ server.startup();
+ return server;
+ }
+
+ public static class SuccessException extends Exception {
+ private static final long serialVersionUID = 1L;
+ }
+
+
+ // ----------------------- Debugging utilities --------------------
+
+ /**
+ * Read topic to list, only using Kafka code.
+ * @return
+ */
+ private static List<MessageAndMetadata<byte[], byte[]>> readTopicToList(String topicName, ConsumerConfig config, final int stopAfter) {
+ ConsumerConnector consumerConnector = Consumer.createJavaConsumerConnector(config);
+ // we request only one stream per consumer instance. Kafka will make sure that each consumer group
+ // will see each message only once.
+ Map<String,Integer> topicCountMap = Collections.singletonMap(topicName, 1);
+ Map<String, List<KafkaStream<byte[], byte[]>>> streams = consumerConnector.createMessageStreams(topicCountMap);
+ if(streams.size() != 1) {
+ throw new RuntimeException("Expected only one message stream but got "+streams.size());
+ }
+ List<KafkaStream<byte[], byte[]>> kafkaStreams = streams.get(topicName);
+ if(kafkaStreams == null) {
+ throw new RuntimeException("Requested stream not available. Available streams: "+streams.toString());
+ }
+ if(kafkaStreams.size() != 1) {
+ throw new RuntimeException("Requested 1 stream from Kafka, bot got "+kafkaStreams.size()+" streams");
+ }
+ LOG.info("Opening Consumer instance for topic '{}' on group '{}'", topicName, config.groupId());
+ ConsumerIterator<byte[], byte[]> iteratorToRead = kafkaStreams.get(0).iterator();
+
+ List<MessageAndMetadata<byte[], byte[]>> result = new ArrayList<MessageAndMetadata<byte[], byte[]>>();
+ int read = 0;
+ while(iteratorToRead.hasNext()) {
+ read++;
+ result.add(iteratorToRead.next());
+ if(read == stopAfter) {
+ LOG.info("Read "+read+" elements");
+ return result;
+ }
+ }
+ return result;
+ }
+
+ private static void printTopic(String topicName, ConsumerConfig config, DeserializationSchema deserializationSchema, int stopAfter){
+ List<MessageAndMetadata<byte[], byte[]>> contents = readTopicToList(topicName, config, stopAfter);
+ LOG.info("Printing contents of topic {} in consumer grouo {}", topicName, config.groupId());
+ for(MessageAndMetadata<byte[], byte[]> message: contents) {
+ Object out = deserializationSchema.deserialize(message.message());
+ LOG.info("Message: partition: {} offset: {} msg: {}", message.partition(), message.offset(), out.toString());
+ }
+ }
+
+ private static void printTopic(String topicName, int elements, ExecutionConfig ec) {
+ // write the sequence to log for debugging purposes
+ Properties stdProps = standardCC.props().props();
+ Properties newProps = new Properties(stdProps);
+ newProps.setProperty("group.id", "topic-printer"+UUID.randomUUID().toString());
+ newProps.setProperty("auto.offset.reset", "smallest");
+ newProps.setProperty("zookeeper.connect", standardCC.zkConnect());
+
+ ConsumerConfig printerConfig = new ConsumerConfig(newProps);
+ DeserializationSchema deserializer = new Utils.TypeInformationSerializationSchema<Tuple2<Integer, Integer>>(new Tuple2<Integer, Integer>(1,1), ec);
+ printTopic(topicName, printerConfig, deserializer, elements);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/kafka/util/KafkaLocalSystemTime.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/kafka/util/KafkaLocalSystemTime.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/kafka/util/KafkaLocalSystemTime.java
new file mode 100644
index 0000000..18fa46f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/kafka/util/KafkaLocalSystemTime.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import kafka.utils.Time;
+
+public class KafkaLocalSystemTime implements Time {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaLocalSystemTime.class);
+
+ @Override
+ public long milliseconds() {
+ return System.currentTimeMillis();
+ }
+
+ public long nanoseconds() {
+ return System.nanoTime();
+ }
+
+ @Override
+ public void sleep(long ms) {
+ try {
+ Thread.sleep(ms);
+ } catch (InterruptedException e) {
+ LOG.warn("Interruption", e);
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/resources/log4j-test.properties b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/resources/log4j-test.properties
new file mode 100644
index 0000000..9ede613
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/resources/log4j-test.properties
@@ -0,0 +1,27 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+log4j.rootLogger=INFO, testlogger
+
+log4j.appender.testlogger=org.apache.log4j.ConsoleAppender
+log4j.appender.testlogger.target = System.err
+log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout
+log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+
+# suppress the irrelevant (wrong) warnings from the netty channel handler
+log4j.logger.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, testlogger
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/resources/logback-test.xml b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..45b3b92
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/src/test/resources/logback-test.xml
@@ -0,0 +1,30 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT"/>
+ </root>
+ <logger name="org.apache.flink.streaming" level="WARN"/>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/pom.xml
new file mode 100644
index 0000000..ebb0af9
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/pom.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-connectors-parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-connector-rabbitmq</artifactId>
+ <name>flink-connector-rabbitmq</name>
+
+ <packaging>jar</packaging>
+
+ <!-- Allow users to pass custom connector versions -->
+ <properties>
+ <rabbitmq.version>3.3.1</rabbitmq.version>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.rabbitmq</groupId>
+ <artifactId>amqp-client</artifactId>
+ <version>${rabbitmq.version}</version>
+ </dependency>
+
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
new file mode 100644
index 0000000..fa729d6
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.rabbitmq;
+
+import java.io.IOException;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.flink.streaming.util.serialization.SerializationSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+
+public class RMQSink<IN> extends RichSinkFunction<IN> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(RMQSink.class);
+
+ private String QUEUE_NAME;
+ private String HOST_NAME;
+ private transient ConnectionFactory factory;
+ private transient Connection connection;
+ private transient Channel channel;
+ private SerializationSchema<IN, byte[]> schema;
+
+ public RMQSink(String HOST_NAME, String QUEUE_NAME, SerializationSchema<IN, byte[]> schema) {
+ this.HOST_NAME = HOST_NAME;
+ this.QUEUE_NAME = QUEUE_NAME;
+ this.schema = schema;
+ }
+
+ /**
+ * Initializes the connection to RMQ.
+ */
+ public void initializeConnection() {
+ factory = new ConnectionFactory();
+ factory.setHost(HOST_NAME);
+ try {
+ connection = factory.newConnection();
+ channel = connection.createChannel();
+ channel.queueDeclare(QUEUE_NAME, false, false, false, null);
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Called when new data arrives to the sink, and forwards it to RMQ.
+ *
+ * @param value
+ * The incoming data
+ */
+ @Override
+ public void invoke(IN value) {
+ try {
+ byte[] msg = schema.serialize(value);
+
+ channel.basicPublish("", QUEUE_NAME, null, msg);
+
+ } catch (IOException e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Cannot send RMQ message {} at {}", QUEUE_NAME, HOST_NAME);
+ }
+ }
+
+ }
+
+ /**
+ * Closes the connection.
+ */
+ private void closeChannel() {
+ try {
+ channel.close();
+ connection.close();
+ } catch (IOException e) {
+ throw new RuntimeException("Error while closing RMQ connection with " + QUEUE_NAME
+ + " at " + HOST_NAME, e);
+ }
+
+ }
+
+ @Override
+ public void open(Configuration config) {
+ initializeConnection();
+ }
+
+ @Override
+ public void close() {
+ closeChannel();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
new file mode 100644
index 0000000..a4c833e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.rabbitmq;
+
+import java.io.IOException;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.source.ConnectorSource;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+
+public class RMQSource<OUT> extends ConnectorSource<OUT> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(RMQSource.class);
+
+ private final String QUEUE_NAME;
+ private final String HOST_NAME;
+
+ private transient ConnectionFactory factory;
+ private transient Connection connection;
+ private transient Channel channel;
+ private transient QueueingConsumer consumer;
+ private transient QueueingConsumer.Delivery delivery;
+
+ private volatile boolean isRunning = false;
+
+ OUT out;
+
+ public RMQSource(String HOST_NAME, String QUEUE_NAME,
+ DeserializationSchema<OUT> deserializationSchema) {
+ super(deserializationSchema);
+ this.HOST_NAME = HOST_NAME;
+ this.QUEUE_NAME = QUEUE_NAME;
+ }
+
+ /**
+ * Initializes the connection to RMQ.
+ */
+ private void initializeConnection() {
+ factory = new ConnectionFactory();
+ factory.setHost(HOST_NAME);
+ try {
+ connection = factory.newConnection();
+ channel = connection.createChannel();
+ channel.queueDeclare(QUEUE_NAME, false, false, false, null);
+ consumer = new QueueingConsumer(channel);
+ channel.basicConsume(QUEUE_NAME, true, consumer);
+ } catch (IOException e) {
+ throw new RuntimeException("Cannot create RMQ connection with " + QUEUE_NAME + " at "
+ + HOST_NAME, e);
+ }
+ }
+
+ @Override
+ public void open(Configuration config) throws Exception {
+ initializeConnection();
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ try {
+ connection.close();
+ } catch (IOException e) {
+ throw new RuntimeException("Error while closing RMQ connection with " + QUEUE_NAME
+ + " at " + HOST_NAME, e);
+ }
+ }
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ if (out != null) {
+ return true;
+ }
+ try {
+ delivery = consumer.nextDelivery();
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Cannot recieve RMQ message {} at {}", QUEUE_NAME, HOST_NAME);
+ }
+ }
+
+ out = schema.deserialize(delivery.getBody());
+ if (schema.isEndOfStream(out)) {
+ out = null;
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public OUT next() throws Exception {
+ if (out != null) {
+ OUT result = out;
+ out = null;
+ return result;
+ }
+
+ try {
+ delivery = consumer.nextDelivery();
+ } catch (Exception e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Cannot recieve RMQ message {} at {}", QUEUE_NAME, HOST_NAME);
+ }
+ }
+
+ out = schema.deserialize(delivery.getBody());
+ if (schema.isEndOfStream(out)) {
+ throw new RuntimeException("RMQ source is at end.");
+ }
+ OUT result = out;
+ out = null;
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
new file mode 100644
index 0000000..0f06235
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-rabbitmq/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.rabbitmq;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.util.serialization.SerializationSchema;
+import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
+
+public class RMQTopology {
+
+ public static void main(String[] args) throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+
+ @SuppressWarnings("unused")
+ DataStream<String> dataStream1 = env.addSource(
+ new RMQSource<String>("localhost", "hello", new SimpleStringSchema())).print();
+
+ @SuppressWarnings("unused")
+ DataStream<String> dataStream2 = env.fromElements("one", "two", "three", "four", "five",
+ "q").addSink(
+ new RMQSink<String>("localhost", "hello", new StringToByteSerializer()));
+
+ env.execute();
+ }
+
+ public static class StringToByteSerializer implements SerializationSchema<String, byte[]> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public byte[] serialize(String element) {
+ return element.getBytes();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/pom.xml
new file mode 100644
index 0000000..871741e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/pom.xml
@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-connectors-parent</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-connector-twitter</artifactId>
+ <name>flink-connector-twitter</name>
+
+ <packaging>jar</packaging>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>hbc-core</artifactId>
+ <version>2.2.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.sling</groupId>
+ <artifactId>org.apache.sling.commons.json</artifactId>
+ <version>2.0.6</version>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <!-- Override artifactSet configuration to build fat-jar with all dependencies packed. -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <configuration>
+ <artifactSet>
+ <includes combine.children="append">
+ <!-- We include all dependencies that transitively depend on guava -->
+ <include>com.twitter:hbc-core</include>
+ <include>com.twitter:joauth</include>
+ </includes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
new file mode 100644
index 0000000..0f16541
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/json/JSONParseFlatMap.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.json;
+
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
+import org.apache.sling.commons.json.JSONException;
+
+/**
+ * Abstract class derived from {@link RichFlatMapFunction} to handle JSON files.
+ *
+ * @param <IN>
+ * Type of the input elements.
+ * @param <OUT>
+ * Type of the returned elements.
+ */
+public abstract class JSONParseFlatMap<IN, OUT> extends RichFlatMapFunction<IN, OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+ // private static final Log LOG = LogFactory.getLog(JSONParseFlatMap.class);
+
+ /**
+ * Get the value object associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public Object get(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).get("retValue");
+ }
+
+ /**
+ * Get the boolean value associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public boolean getBoolean(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).getBoolean("retValue");
+ }
+
+ /**
+ * Get the double value associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public double getDouble(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).getDouble("retValue");
+ }
+
+ /**
+ * Get the int value associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public int getInt(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).getInt("retValue");
+ }
+
+ /**
+ * Get the long value associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public long getLong(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).getLong("retValue");
+ }
+
+ /**
+ * Get the String value associated with a key form a JSON code. It can find
+ * embedded fields, too.
+ *
+ * @param jsonText
+ * JSON String in which the field is searched.
+ * @param field
+ * The key whose value is searched for.
+ * @return The object associated with the field.
+ * @throws JSONException
+ * If the field is not found.
+ */
+ public String getString(String jsonText, String field) throws JSONException {
+ JSONParser parser = new JSONParser(jsonText);
+
+ return parser.parse(field).getString("retValue");
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
new file mode 100644
index 0000000..c1eabbd
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/json/JSONParser.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.json;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.StringTokenizer;
+
+import org.apache.sling.commons.json.JSONArray;
+import org.apache.sling.commons.json.JSONException;
+import org.apache.sling.commons.json.JSONObject;
+
+/**
+ * A JSONParser contains a JSONObject and provides opportunity to access
+ * embedded fields in JSON code.
+ */
+public class JSONParser {
+
+ private JSONObject originalJO;
+ private String searchedfield;
+ private Object temp;
+
+ /**
+ * Construct a JSONParser from a string. The string has to be a JSON code
+ * from which we want to get a field.
+ *
+ * @param jsonText
+ * A string which contains a JSON code. String representation of
+ * a JSON code.
+ * @throws JSONException
+ * If there is a syntax error in the source string.
+ */
+ public JSONParser(String jsonText) throws JSONException {
+ originalJO = new JSONObject(jsonText);
+ }
+
+ /**
+ *
+ * Parse the JSON code passed to the constructor to find the given key.
+ *
+ * @param key
+ * The key whose value is searched for.
+ * @return A JSONObject which has only one field called "retValue" and the
+ * value associated to it is the searched value. The methods of
+ * JSONObject can be used to get the field value in a desired
+ * format.
+ * @throws JSONException
+ * If the key is not found.
+ */
+ public JSONObject parse(String key) throws JSONException {
+ initializeParser(key);
+ parsing();
+ return putResultInJSONObj();
+ }
+
+ /**
+ * Prepare the fields of the class for the parsing
+ *
+ * @param key
+ * The key whose value is searched for.
+ * @throws JSONException
+ * If the key is not found.
+ */
+ private void initializeParser(String key) throws JSONException {
+ searchedfield = key;
+ temp = new JSONObject(originalJO.toString());
+ }
+
+ /**
+ * This function goes through the given field and calls the appropriate
+ * functions to treat the units between the punctuation marks.
+ *
+ * @throws JSONException
+ * If the key is not found.
+ */
+ private void parsing() throws JSONException {
+ StringTokenizer st = new StringTokenizer(searchedfield, ".");
+ while (st.hasMoreTokens()) {
+ find(st.nextToken());
+ }
+ }
+
+ /**
+ * Search for the next part of the field and update the state if it was
+ * found.
+ *
+ * @param nextToken
+ * The current part of the searched field.
+ * @throws JSONException
+ * If the key is not found.
+ */
+ private void find(String nextToken) throws JSONException {
+ if (endsWithBracket(nextToken)) {
+ treatAllBracket(nextToken);
+ } else {
+ temp = ((JSONObject) temp).get(nextToken);
+ }
+ }
+
+ /**
+ * Determine whether the given string ends with a closing square bracket ']'
+ *
+ * @param nextToken
+ * The current part of the searched field.
+ * @return True if the given string ends with a closing square bracket ']'
+ * and false otherwise.
+ */
+ private boolean endsWithBracket(String nextToken) {
+ return nextToken.substring(nextToken.length() - 1).endsWith("]");
+ }
+
+ /**
+ * Handle (multidimensional) arrays. Treat the square bracket pairs one
+ * after the other if necessary.
+ *
+ * @param nextToken
+ * The current part of the searched field.
+ * @throws JSONException
+ * If the searched element is not found.
+ */
+ private void treatAllBracket(String nextToken) throws JSONException {
+ List<String> list = Arrays.asList(nextToken.split("\\["));
+ ListIterator<String> iter = list.listIterator();
+
+ temp = ((JSONObject) temp).get(iter.next());
+
+ while (iter.hasNext()) {
+ int index = Integer.parseInt(cutBracket(iter.next()));
+ temp = ((JSONArray) temp).get(index);
+ }
+ }
+
+ /**
+ * Remove the last character of the string.
+ *
+ * @param string
+ * String to modify.
+ * @return The given string without the last character.
+ */
+ private String cutBracket(String string) {
+ return string.substring(0, string.length() - 1);
+ }
+
+ /**
+ * Save the result of the search into a JSONObject.
+ *
+ * @return A special JSONObject which contain only one key. The value
+ * associated to this key is the result of the search.
+ * @throws JSONException
+ * If there is a problem creating the JSONObject. (e.g. invalid
+ * syntax)
+ */
+ private JSONObject putResultInJSONObj() throws JSONException {
+ JSONObject jo = new JSONObject();
+ jo.put("retValue", temp);
+ return jo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
new file mode 100644
index 0000000..78e4aa5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.twitter;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.twitter.hbc.ClientBuilder;
+import com.twitter.hbc.core.Constants;
+import com.twitter.hbc.core.endpoint.StatusesSampleEndpoint;
+import com.twitter.hbc.core.processor.StringDelimitedProcessor;
+import com.twitter.hbc.httpclient.BasicClient;
+import com.twitter.hbc.httpclient.auth.Authentication;
+import com.twitter.hbc.httpclient.auth.OAuth1;
+
+/**
+ * Implementation of {@link SourceFunction} specialized to emit tweets from
+ * Twitter. It can connect to Twitter Streaming API, collect tweets and
+ */
+public class TwitterSource extends RichParallelSourceFunction<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TwitterSource.class);
+
+ private static final long serialVersionUID = 1L;
+ private String authPath;
+ private transient BlockingQueue<String> queue;
+ private int queueSize = 10000;
+ private transient BasicClient client;
+ private int waitSec = 5;
+
+ private int maxNumberOfTweets;
+ private int currentNumberOfTweets;
+
+ private String nextElement = null;
+
+ private volatile boolean isRunning = false;
+
+ /**
+ * Create {@link TwitterSource} for streaming
+ *
+ * @param authPath
+ * Location of the properties file containing the required
+ * authentication information.
+ */
+ public TwitterSource(String authPath) {
+ this.authPath = authPath;
+ maxNumberOfTweets = -1;
+ }
+
+ /**
+ * Create {@link TwitterSource} to collect finite number of tweets
+ *
+ * @param authPath
+ * Location of the properties file containing the required
+ * authentication information.
+ * @param numberOfTweets
+ *
+ */
+ public TwitterSource(String authPath, int numberOfTweets) {
+ this.authPath = authPath;
+ this.maxNumberOfTweets = numberOfTweets;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ initializeConnection();
+ currentNumberOfTweets = 0;
+ }
+
+ /**
+ * Initialize Hosebird Client to be able to consume Twitter's Streaming API
+ */
+ private void initializeConnection() {
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Initializing Twitter Streaming API connection");
+ }
+
+ queue = new LinkedBlockingQueue<String>(queueSize);
+
+ StatusesSampleEndpoint endpoint = new StatusesSampleEndpoint();
+ endpoint.stallWarnings(false);
+
+ Authentication auth = authenticate();
+
+ initializeClient(endpoint, auth);
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Twitter Streaming API connection established successfully");
+ }
+ }
+
+ private OAuth1 authenticate() {
+
+ Properties authenticationProperties = loadAuthenticationProperties();
+
+ return new OAuth1(authenticationProperties.getProperty("consumerKey"),
+ authenticationProperties.getProperty("consumerSecret"),
+ authenticationProperties.getProperty("token"),
+ authenticationProperties.getProperty("secret"));
+ }
+
+ /**
+ * Reads the given properties file for the authentication data.
+ *
+ * @return the authentication data.
+ */
+ private Properties loadAuthenticationProperties() {
+ Properties properties = new Properties();
+ try {
+ InputStream input = new FileInputStream(authPath);
+ properties.load(input);
+ input.close();
+ } catch (Exception e) {
+ throw new RuntimeException("Cannot open .properties file: " + authPath, e);
+ }
+ return properties;
+ }
+
+ private void initializeClient(StatusesSampleEndpoint endpoint, Authentication auth) {
+
+ client = new ClientBuilder().name("twitterSourceClient").hosts(Constants.STREAM_HOST)
+ .endpoint(endpoint).authentication(auth)
+ .processor(new StringDelimitedProcessor(queue)).build();
+
+ client.connect();
+ }
+
+ /**
+ * Put tweets into output
+ *
+ * @param collector
+ * Collector in which the tweets are collected.
+ */
+ protected void collectFiniteMessages(Collector<String> collector) {
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Collecting tweets");
+ }
+
+ for (int i = 0; i < maxNumberOfTweets; i++) {
+ collectOneMessage(collector);
+ }
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Collecting tweets finished");
+ }
+ }
+
+ /**
+ * Put tweets into output
+ *
+ * @param collector
+ * Collector in which the tweets are collected.
+ */
+ protected void collectMessages(Collector<String> collector) {
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Tweet-stream begins");
+ }
+
+ while (isRunning) {
+ collectOneMessage(collector);
+ }
+ }
+
+ /**
+ * Put one tweet into the output.
+ *
+ * @param collector
+ * Collector in which the tweets are collected.
+ */
+ protected void collectOneMessage(Collector<String> collector) {
+ if (client.isDone()) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Client connection closed unexpectedly: {}", client.getExitEvent()
+ .getMessage());
+ }
+ }
+
+ try {
+ String msg = queue.poll(waitSec, TimeUnit.SECONDS);
+ if (msg != null) {
+ collector.collect(msg);
+ } else {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Did not receive a message in {} seconds", waitSec);
+ }
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException("'Waiting for tweet' thread is interrupted", e);
+ }
+
+ }
+
+ private void closeConnection() {
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Initiating connection close");
+ }
+
+ client.stop();
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Connection closed successfully");
+ }
+ }
+
+ /**
+ * Get the size of the queue in which the tweets are contained temporarily.
+ *
+ * @return the size of the queue in which the tweets are contained
+ * temporarily
+ */
+ public int getQueueSize() {
+ return queueSize;
+ }
+
+ /**
+ * Set the size of the queue in which the tweets are contained temporarily.
+ *
+ * @param queueSize
+ * The desired value.
+ */
+ public void setQueueSize(int queueSize) {
+ this.queueSize = queueSize;
+ }
+
+ /**
+ * This function tells how long TwitterSource waits for the tweets.
+ *
+ * @return Number of second.
+ */
+ public int getWaitSec() {
+ return waitSec;
+ }
+
+ /**
+ * This function sets how long TwitterSource should wait for the tweets.
+ *
+ * @param waitSec
+ * The desired value.
+ */
+ public void setWaitSec(int waitSec) {
+ this.waitSec = waitSec;
+ }
+
+ @Override
+ public boolean reachedEnd() throws Exception {
+ if (currentNumberOfTweets >= maxNumberOfTweets) {
+ return false;
+ }
+
+ if (nextElement != null) {
+ return true;
+ }
+ if (client.isDone()) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Client connection closed unexpectedly: {}", client.getExitEvent()
+ .getMessage());
+ }
+ return false;
+ }
+
+ try {
+ String msg = queue.poll(waitSec, TimeUnit.SECONDS);
+ if (msg != null) {
+ nextElement = msg;
+ return true;
+ } else {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Did not receive a message in {} seconds", waitSec);
+ }
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException("'Waiting for tweet' thread is interrupted", e);
+ }
+ return false;
+ }
+
+ @Override
+ public String next() throws Exception {
+ if (nextElement != null) {
+ String result = nextElement;
+ nextElement = null;
+ return result;
+ }
+ if (reachedEnd()) {
+ throw new RuntimeException("Twitter stream end reached.");
+ } else {
+ String result = nextElement;
+ nextElement = null;
+ return result;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/665bcec7/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
new file mode 100644
index 0000000..e500fef
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-twitter/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterStreaming.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.twitter;
+
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.SinkFunction;
+import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
+import org.apache.flink.util.Collector;
+import org.apache.sling.commons.json.JSONException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TwitterStreaming {
+
+ private static final int PARALLELISM = 1;
+ private static final int SOURCE_PARALLELISM = 1;
+ private static final int NUMBEROFTWEETS = 100;
+
+ private static final Logger LOG = LoggerFactory.getLogger(TwitterStreaming.class);
+
+ public static class TwitterSink implements SinkFunction<Tuple5<Long, Integer, String, String, String>> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void invoke(Tuple5<Long, Integer, String, String, String> tuple) {
+ System.out.println("ID: " + tuple.f0 + " int: " + tuple.f1 + " LANGUAGE: " + tuple.f2);
+ System.out.println("NAME: " + tuple.f4);
+ System.out.println("TEXT: " + tuple.f3);
+ System.out.println("");
+ }
+
+ }
+
+ public static class SelectDataFlatMap extends
+ JSONParseFlatMap<String, Tuple5<Long, Integer, String, String, String>> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void flatMap(String value, Collector<Tuple5<Long, Integer, String, String, String>> out)
+ throws Exception {
+ try {
+ out.collect(new Tuple5<Long, Integer, String, String, String>(
+ getLong(value, "id"),
+ getInt(value, "entities.hashtags[0].indices[1]"),
+ getString(value, "lang"),
+ getString(value, "text"),
+ getString(value, "user.name")));
+ } catch (JSONException e) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("Field not found");
+ }
+ }
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ String path = new String();
+
+ if (args != null && args.length == 1) {
+ path = args[0];
+ } else {
+ System.err.println("USAGE:\nTwitterStreaming <pathToPropertiesFile>");
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment
+ .createLocalEnvironment(PARALLELISM);
+
+ DataStream<String> streamSource = env.addSource(new TwitterSource(path, NUMBEROFTWEETS))
+ .setParallelism(SOURCE_PARALLELISM);
+
+ DataStream<Tuple5<Long, Integer, String, String, String>> selectedDataStream = streamSource
+ .flatMap(new SelectDataFlatMap());
+
+ selectedDataStream.addSink(new TwitterSink());
+
+ env.execute();
+ }
+}